{"methodVersion":"benchmark-meta-score-v1","areas":["coding","reasoning","math","knowledge","agentic","general","finance","legal","health","cybersecurity","multimodal","search","productivity","education"],"familyCount":71,"assessedObservations":27017,"includedObservations":3187,"excluded":{"alternative-reasoning-tier":708,"ambiguous-model":349,"composite":997,"inactive":335,"incomparable-setup":93,"insufficient-peers":48,"invalid-value":1,"subtask":14150,"unclassified-family":776,"unreviewed-source":6373},"models":[{"modelId":"agnes-3.0-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"agnes-3.0-flash-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ai21-jamba-large","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ai21-jamba-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"aion-3.5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"aion-3.5-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"aisingapore-gemma-sea-lion-v4-27b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen2-5-vl-72b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen2.5-coder-0.5b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen2.5-coder-32b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen3-coder-30b-a3b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen3-coder-480b-a35b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen3-vl-235b-a22b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwen3.8-flash-next","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"alibaba-qwq-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"aliceai-foundation-80b-a3b-base","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"amazon-nova-2-lite","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"amazon-nova-lite","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"amazon-nova-micro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"amazon-nova-premier","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"amazon-nova-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"apodex-1.1-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"arcee-ai-trinity-large-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"arcee-ai-trinity-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"arcee-ai-trinity-nano-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"arex-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"atria-dawn-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"chat-latest","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"claude-fable-5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"research:claude-fable-5:9","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":1,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-fable-5_high. Epoch-Modellbezeichnung: Claude Fable 5. Anzeigenamenfeld: Claude Fable 5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: hwrnvtEbDAocFtjcsHqsQh. Startzeit: 2026-08-06T22:18:19.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=1.0 und stderr=0.0. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: claude-fable-5_high; Epoch model label: Claude Fable 5; Display label field: Claude Fable 5 (high); Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: hwrnvtEbDAocFtjcsHqsQh. Started at 2026-08-06T22:18:19.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=1.0 and stderr=0.0. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":73.6842105263158,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":48.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.6842105263158,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 48,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 48.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":1539.25,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":82.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 82,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 82.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":54.071301140653816,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 54,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 54.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":24,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 24,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 24.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":1609.83,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":49.27536231884058,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":63.469387755102005,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.27536231884058,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 63,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 63.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":43.3,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"enigma-eval","area":"reasoning","label":{"de":"EnigmaEval","en":"EnigmaEval"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-enigma-eval-2026-07-23-claude-fable-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/enigma_eval","benchmarkId":"enigma-eval","benchmarkVersion":"open-source release 2026-07-23","comparisonCohort":"scale-enigma-eval-2026-07-23","retrievedAt":"2026-08-26","value":39.28,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":5,"methodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Kennung des Quellmodells: curated/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-high. Evaluator: Scale AI. Datensatztyp: public. Stand der Rangliste: 2026-07-23. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 39,28 % ± 2,8. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-de06ddcfa8a8. Prüfsumme der Quelldaten: de06ddcfa8a81a2242bdcf8cae4bc1d2e0d1d9e896cb450a19ce0227b96075a7. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Source model ID: curated/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-high. Evaluator: Scale AI. Dataset type: public. Leaderboard date: 2026-07-23. Attempts: pass@1. Reasoning effort: high. Score display: 39.28% ± 2.8. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-de06ddcfa8a8. Source payload hash: de06ddcfa8a81a2242bdcf8cae4bc1d2e0d1d9e896cb450a19ce0227b96075a7. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.","en":"Complex multimodal puzzles with exact answer matching and one attempt."},"scoreRunConfiguration":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Versuche: pass@1. Reasoning-Stufe: high","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-enigma-eval-2026-07-23","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":61.904761904761905,"measurements":[{"observationId":"research:claude-fable-5:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.47,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.904761904761905,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-fable-5_max. Epoch-Modellbezeichnung: Claude Fable 5. Anzeigenamenfeld: Claude Fable 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: bxS76CJuVws8iwWU2Wr2PM. Startzeit: 2026-10-13T13:34:37.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.47 und stderr=0.05016135580465918. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-fable-5_max; Epoch model label: Claude Fable 5; Display label field: Claude Fable 5 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: bxS76CJuVws8iwWU2Wr2PM. Started at 2026-10-13T13:34:37.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.47 and stderr=0.05016135580465918. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":90.4040404040404,"measurements":[{"observationId":"research:claude-fable-5:1","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.902,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":94.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-fable-5_max. Epoch-Modellbezeichnung: Claude Fable 5. Anzeigenamenfeld: Claude Fable 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: ReDAtJGGfRtrEGGWf4Zbsd. Startzeit: 2026-06-09T18:12:35.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.902 und stderr=0.046. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-fable-5_max; Epoch model label: Claude Fable 5; Display label field: Claude Fable 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: ReDAtJGGfRtrEGGWf4Zbsd. Started at 2026-06-09T18:12:35.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.902 and stderr=0.046. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:claude-fable-5:0","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.8701754385964913,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":86.36363636363636,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-fable-5_max. Epoch-Modellbezeichnung: Claude Fable 5. Anzeigenamenfeld: Claude Fable 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: E28uZiJvrPgKtawbJpoZQ8. Startzeit: 2026-06-09T18:12:35.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.8701754385964913 und stderr=0.019944477920133024. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-fable-5_max; Epoch model label: Claude Fable 5; Display label field: Claude Fable 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: E28uZiJvrPgKtawbJpoZQ8. Started at 2026-06-09T18:12:35.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.8701754385964913 and stderr=0.019944477920133024. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":48.51190476190476,"measurements":[{"observationId":"research:claude-fable-5:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.8585858585858586,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":9.523809523809524,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-fable-5_max. Epoch-Modellbezeichnung: Claude Fable 5. Anzeigenamenfeld: Claude Fable 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: S5pm7ctk9zgPfbyHrbD9AV. Startzeit: 2026-08-06T22:17:56.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.8585858585858586 und stderr=0.024825909793343353. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-fable-5_max; Epoch model label: Claude Fable 5; Display label field: Claude Fable 5 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: S5pm7ctk9zgPfbyHrbD9AV. Started at 2026-08-06T22:17:56.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.8585858585858586 and stderr=0.024825909793343353. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.182,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.5,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 99.902. Standardfehler: 1.939. Kosten je Aufgabe in USD: 0.450473. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 99.902. Standard error: 1.939. Cost per task in USD: 0.450473. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":55.468025949953706,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 55,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 55.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"lmarena:webdev:overall:claude-fable-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1623.8279012561834,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":66.66666666666667,"participants":37,"methodology":{"de":"Arena-Bewertung aus 15.295 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5-high. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 15,295 published pairwise votes.. Source model ID: lmarena/claude-fable-5-high. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 15.295 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 15,295 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 15.295 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 15,295 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"lmarena:search:overall:claude-fable-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1229.1397516536,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":53.84615384615385,"participants":14,"methodology":{"de":"Arena-Bewertung aus 73.596 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5-high. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 73,596 published pairwise votes.. Source model ID: lmarena/claude-fable-5-high. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 73.596 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: high","en":"Arena rating from 73,596 published pairwise votes.. Harness: Arena Search. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 73.596 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: high","en":"Arena rating from 73,596 published pairwise votes.. Harness: Arena Search. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":94.44444444444444,"measurements":[{"observationId":"lmarena:overall:claude-fable-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1503.3770704268557,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":94.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 41.439 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5-high. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 41,439 published pairwise votes.. Source model ID: lmarena/claude-fable-5-high. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 41.439 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 41,439 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 41.439 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 41,439 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-fable-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1307.9658540715964,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":100,"participants":31,"methodology":{"de":"Arena-Bewertung aus 14.175 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5-high. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 14,175 published pairwise votes.. Source model ID: lmarena/claude-fable-5-high. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 14.175 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 14,175 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 14.175 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 14,175 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":95.74468085106383,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":89.306,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.74468085106383,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 61.44. Standardfehler: 0.743. Kosten je Aufgabe in USD: 0.16064. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 61.44. Standard error: 0.743. Cost per task in USD: 0.16064. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-prbench-finance-claude-fable-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":53.86,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-max. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 53,86 % ± 0,14. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-bf274b71f97c. Prüfsumme der Quelldaten: bf274b71f97c40f0911be8c49cc3640b9020b7e8f195d1da9823e3c03787cbe3. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-max. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 53.86% ± 0.14. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-bf274b71f97c. Source payload hash: bf274b71f97c40f0911be8c49cc3640b9020b7e8f195d1da9823e3c03787cbe3. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: max","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-prbench-legal-claude-fable-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":52.56,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5-max. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 52,56 % ± 0,54. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-6eb497f65483. Prüfsumme der Quelldaten: 6eb497f65483f7d7f740b0e8e20ca95ea1a57c2bd1a03575b3448bb5e468d88f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: claude-fable-5-max. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 52.56% ± 0.54. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-6eb497f65483. Source payload hash: 6eb497f65483f7d7f740b0e8e20ca95ea1a57c2bd1a03575b3448bb5e468d88f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: max","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":91.52542372881356,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":95,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.52542372881356,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 356.182. Standardfehler: 0.976. Kosten je Aufgabe in USD: 2.046736. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 356.182. Standard error: 0.976. Cost per task in USD: 2.046736. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":87.53060551841041,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:cd55210d-358e-4df1-ba9c-9acb5f186cc9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max%2C%20Opus%204.8%20Fallback","retrievedAt":"2026-10-19","value":42.4242424242424,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback. Angezeigter Wert: 42,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd55210d-358e-4df1-ba9c-9acb5f186cc9/claude-fable-5/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5 (Max, Opus 4.8 Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback. Score display: 42.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5 (Max, Opus 4.8 Fallback) (Release claude-fable-5, Zeile claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Opus 4.8 Fallback","en":"Exact AA configuration: Claude Fable 5 (Max, Opus 4.8 Fallback) (release claude-fable-5, row claude-fable-5, UUID cd55210d-358e-4df1-ba9c-9acb5f186cc9, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Opus 4.8 Fallback). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max, Opus 4.8 Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max, Opus 4.8 Fallback","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":80.524,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.92857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 504.537. Standardfehler: 1.35. Kosten je Aufgabe in USD: 1.429025. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 504.537. Standard error: 1.35. Cost per task in USD: 1.429025. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":41.414,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.67567567567568,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4093.25. Standardfehler: 1.336. Kosten je Aufgabe in USD: 30.339756. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4093.25. Standard error: 1.336. Cost per task in USD: 30.339756. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":67.74193548387096,"measurements":[{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":15.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.74193548387096,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7157.669. Standardfehler: 4.381. Kosten je Aufgabe in USD: 51.990885. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7157.669. Standard error: 4.381. Cost per task in USD: 51.990885. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":87.09677419354838,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":55.065,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.09677419354838,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6559.388. Standardfehler: 4.605. Kosten je Aufgabe in USD: 112.098109. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6559.388. Standard error: 4.605. Cost per task in USD: 112.098109. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":98.64864864864865,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":71.834,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.64864864864865,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 52.99. Standardfehler: 0.883. Kosten je Aufgabe in USD: 1.737573. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 52.99. Standard error: 0.883. Cost per task in USD: 1.737573. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":94.91525423728814,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":73.666,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.91525423728814,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1594.153. Standardfehler: 2.468. Kosten je Aufgabe in USD: 12.354419. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1594.153. Standard error: 2.468. Cost per task in USD: 12.354419. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":81.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":56.314,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 610.952. Standardfehler: 0.84. Kosten je Aufgabe in USD: 8.061563. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 610.952. Standard error: 0.84. Cost per task in USD: 8.061563. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":98.7012987012987,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":89.778,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.7012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 118.53. Standardfehler: 0.892. Kosten je Aufgabe in USD: 0.42816. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 118.53. Standard error: 0.892. Cost per task in USD: 0.42816. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":86.06557377049181,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":11.25,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.06557377049181,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1613.035. Standardfehler: 2.148. Kosten je Aufgabe in USD: 19.225253. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1613.035. Standard error: 2.148. Cost per task in USD: 19.225253. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":88.561,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8.964. Standardfehler: 0.331. Kosten je Aufgabe in USD: 0.030639. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 8.964. Standard error: 0.331. Cost per task in USD: 0.030639. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":49.519,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1337.604. Standardfehler: 3.475. Kosten je Aufgabe in USD: 9.791426. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1337.604. Standard error: 3.475. Cost per task in USD: 9.791426. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":97.01492537313433,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":56.07,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.01492537313433,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 91.439. Standardfehler: 2.203. Kosten je Aufgabe in USD: 0.591071. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 91.439. Standard error: 2.203. Cost per task in USD: 0.591071. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":86.76470588235294,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":88.522,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.76470588235294,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 119.475. Standardfehler: 1.945. Kosten je Aufgabe in USD: 0.583239. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 119.475. Standard error: 1.945. Cost per task in USD: 0.583239. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":97.40259740259741,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":91.502,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.40259740259741,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 24.998. Standardfehler: 0.278. Kosten je Aufgabe in USD: 0.082276. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 24.998. Standard error: 0.278. Cost per task in USD: 0.082276. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":88.46153846153847,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.919,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.46153846153847,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.096. Standardfehler: 0.909. Kosten je Aufgabe in USD: 0.175013. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 16.096. Standard error: 0.909. Cost per task in USD: 0.175013. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":83.6734693877551,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":2,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.6734693877551,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9447.46. Standardfehler: 0.992. Kosten je Aufgabe in USD: 75.675797. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 9447.46. Standard error: 0.992. Cost per task in USD: 75.675797. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":95,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 822.739. Standardfehler: 2.19. Kosten je Aufgabe in USD: 5.451518. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 822.739. Standard error: 2.19. Cost per task in USD: 5.451518. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":89.65517241379311,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":51.89,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.65517241379311,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 116.916. Standardfehler: 3.4. Kosten je Aufgabe in USD: 0.65214. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 116.916. Standard error: 3.4. Cost per task in USD: 0.65214. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":89.47368421052632,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":70.433,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.47368421052632,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1276.576. Standardfehler: 1.187. Kosten je Aufgabe in USD: 4.585446. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1276.576. Standard error: 1.187. Cost per task in USD: 4.585446. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":89.0909090909091,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":38.281,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.0909090909091,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 866.628. Standardfehler: 3.503. Kosten je Aufgabe in USD: 6.696087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 866.628. Standard error: 3.503. Cost per task in USD: 6.696087. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":98.6842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":76.942,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.6842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 56.865. Standardfehler: 0.821. Kosten je Aufgabe in USD: 0.220133. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 56.865. Standard error: 0.821. Cost per task in USD: 0.220133. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":97.22222222222223,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-fable-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":90.352,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.22222222222223,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5. Name des Quellmodells: Claude Fable 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3711.026. Standardfehler: 2.097. Kosten je Aufgabe in USD: 41.714694. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5. Source model name: Claude Fable 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3711.026. Standard error: 2.097. Cost per task in USD: 41.714694. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":85.0558406186449,"coveredFamilies":43,"totalFamilies":71,"sourceCount":5,"coverage":0.5916383219954651,"lowerBound":50.322294819528786,"upperBound":91.15846261998229,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.4555555555555557,"lowerBound":39.06241880287836,"upperBound":93.5068632473228,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":79.05580192165559,"coveredFamilies":6,"totalFamilies":7,"sourceCount":4,"coverage":0.7857142857142857,"lowerBound":62.11527293844367,"upperBound":83.5438443670151,"provisional":true,"configurationsDiffer":true},"math":{"score":88.64283864283864,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":53.18570318570318,"upperBound":93.18570318570318,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":98.09154260373774,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":65.39436173582516,"upperBound":98.7276950691585,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":26.872058194266156,"upperBound":93.53872486093283,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":28.743961352657003,"upperBound":88.74396135265701,"provisional":true,"configurationsDiffer":true},"finance":{"score":93.06674680448099,"coveredFamilies":7,"totalFamilies":7,"sourceCount":2,"coverage":1,"lowerBound":93.06674680448099,"upperBound":93.06674680448099,"provisional":true,"configurationsDiffer":true},"legal":{"score":94.43306010928961,"coveredFamilies":4,"totalFamilies":4,"sourceCount":2,"coverage":1,"lowerBound":94.43306010928961,"upperBound":94.43306010928961,"provisional":true,"configurationsDiffer":true},"health":{"score":91.88981562774363,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":61.259877085162415,"upperBound":94.59321041849574,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":36.96808510638298,"upperBound":99.46808510638297,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":53.84615384615385,"upperBound":53.84615384615385,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":44.73684210526316,"upperBound":94.73684210526315,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":44.827586206896555,"upperBound":94.82758620689656,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-fable-5.1","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":96.05263157894737,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:3e87c73e-a257-495e-9730-367a66229811","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":57.49999999999999,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.05263157894737,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 57,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 57.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":93.33333333333333,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:3e87c73e-a257-495e-9730-367a66229811","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":48.9732333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.33333333333333,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 49,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 49.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":88.57142857142857,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:3e87c73e-a257-495e-9730-367a66229811","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":49.5291902071563,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.57142857142857,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 49,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 49.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":90.78947368421052,"measurements":[{"observationId":"vals-gpqa:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.78947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 45.857. Standardfehler: 1.863. Kosten je Aufgabe in USD: 0.204523. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 45.857. Standard error: 1.863. Cost per task in USD: 0.204523. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"lmarena:webdev:overall:claude-fable-5.1-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1742.652412524194,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":88.88888888888889,"participants":37,"methodology":{"de":"Arena-Bewertung aus 7.230 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5.1-max. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5.1-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 7,230 published pairwise votes.. Source model ID: lmarena/claude-fable-5.1-max. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: claude-fable-5.1-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 7.230 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 7,230 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 7.230 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 7,230 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"lmarena:overall:claude-fable-5.1-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1500.567539502662,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":91.66666666666667,"participants":37,"methodology":{"de":"Arena-Bewertung aus 14.121 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5.1-max. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5.1-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 14,121 published pairwise votes.. Source model ID: lmarena/claude-fable-5.1-max. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: claude-fable-5.1-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 14.121 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 14,121 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 14.121 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 14,121 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-fable-5.1-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1286.4074118221424,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":73.33333333333333,"participants":31,"methodology":{"de":"Arena-Bewertung aus 4.648 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-fable-5.1-max. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: claude-fable-5.1-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 4,648 published pairwise votes.. Source model ID: lmarena/claude-fable-5.1-max. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: claude-fable-5.1-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 4.648 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 4,648 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 4.648 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 4,648 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":100,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":90.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 37.47. Standardfehler: 0.7. Kosten je Aufgabe in USD: 0.166285. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 37.47. Standard error: 0.7. Cost per task in USD: 0.166285. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":92.59169884169884,"measurements":[{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":85.019,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.64285714285714,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 500.355. Standardfehler: 0.991. Kosten je Aufgabe in USD: 3.074796. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 500.355. Standard error: 0.991. Cost per task in USD: 3.074796. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":58.081,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.54054054054055,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3623.762. Standardfehler: 3.312. Kosten je Aufgabe in USD: 17.175337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3623.762. Standard error: 3.312. Cost per task in USD: 17.175337. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":89.20056100981768,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:3e87c73e-a257-495e-9730-367a66229811","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":43.3333333333333,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.30434782608695,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Modellstand: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 43,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/3e87c73e-a257-495e-9730-367a66229811/claude-fable-5-1/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Model snapshot: Claude Fable 5.1 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 43.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Fable 5.1 (Max, Default Fallback) (Release claude-fable-5-1, Zeile claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Fable 5.1 (Max, Default Fallback) (release claude-fable-5-1, row claude-fable-5-1, UUID 3e87c73e-a257-495e-9730-367a66229811, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max, Default Fallback","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":40,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.09677419354838,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9376.226. Standardfehler: 5.898. Kosten je Aufgabe in USD: 38.010073. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 9376.226. Standard error: 5.898. Cost per task in USD: 38.010073. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":85.48387096774194,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":54.607,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.48387096774194,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15610.474. Standardfehler: 4.812. Kosten je Aufgabe in USD: 70.973455. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 15610.474. Standard error: 4.812. Cost per task in USD: 70.973455. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":70.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1112.095. Standardfehler: 5.43. Kosten je Aufgabe in USD: 4.010688. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1112.095. Standard error: 5.43. Cost per task in USD: 4.010688. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":76.67,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1771.558. Standardfehler: 2.085. Kosten je Aufgabe in USD: 15.93249. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1771.558. Standard error: 2.085. Cost per task in USD: 15.93249. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":93.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":58.877,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1094.165. Standardfehler: 2.058. Kosten je Aufgabe in USD: 8.346696. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1094.165. Standard error: 2.058. Cost per task in USD: 8.346696. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":87.87878787878788,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":90.778,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.87878787878788,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1805.905. Standardfehler: 4.647. Kosten je Aufgabe in USD: 11.196959. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1805.905. Standard error: 4.647. Cost per task in USD: 11.196959. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":90.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 59.738. Standardfehler: 0.863. Kosten je Aufgabe in USD: 0.260604. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 59.738. Standard error: 0.863. Cost per task in USD: 0.260604. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":70.49180327868852,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":6.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.49180327868852,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6749.446. Standardfehler: 1.174. Kosten je Aufgabe in USD: 46.211522. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6749.446. Standard error: 1.174. Cost per task in USD: 46.211522. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":98.71794871794872,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":88.511,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.71794871794872,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.405. Standardfehler: 0.42. Kosten je Aufgabe in USD: 0.051606. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 12.405. Standard error: 0.42. Cost per task in USD: 0.051606. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":99.16666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":55.288,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":99.16666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3449.382. Standardfehler: 3.456. Kosten je Aufgabe in USD: 23.055789. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3449.382. Standard error: 3.456. Cost per task in USD: 23.055789. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":91.04477611940298,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":53.509,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.04477611940298,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 215.494. Standardfehler: 2.165. Kosten je Aufgabe in USD: 1.116862. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 215.494. Standard error: 2.165. Cost per task in USD: 1.116862. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":98.52941176470588,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":91.294,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.52941176470588,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 188.363. Standardfehler: 1.953. Kosten je Aufgabe in USD: 0.9635. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 188.363. Standard error: 1.953. Cost per task in USD: 0.9635. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":92.375,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.589. Standardfehler: 0.266. Kosten je Aufgabe in USD: 0.097724. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 20.589. Standard error: 0.266. Cost per task in USD: 0.097724. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":98.07692307692308,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":70.786,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.07692307692308,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 14.98. Standardfehler: 0.896. Kosten je Aufgabe in USD: 0.171632. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 14.98. Standard error: 0.896. Cost per task in USD: 0.171632. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":85,"measurements":[{"observationId":"vals-mysterymechanism:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":47.748,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85,"participants":21,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1060.876. Standardfehler: 3.36. Kosten je Aufgabe in USD: 5.631518. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1060.876. Standard error: 3.36. Cost per task in USD: 5.631518. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":97.95918367346938,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":7,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.95918367346938,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8791.194. Standardfehler: 1.809. Kosten je Aufgabe in USD: 58.51711. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 8791.194. Standard error: 1.809. Cost per task in USD: 58.51711. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":97.43589743589743,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":100,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.43589743589743,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 478.303. Standardfehler: 0. Kosten je Aufgabe in USD: 2.70347. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 478.303. Standard error: 0. Cost per task in USD: 2.70347. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":62.06896551724138,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":48.526,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.06896551724138,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 105.745. Standardfehler: 3.336. Kosten je Aufgabe in USD: 0.61029. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 105.745. Standard error: 3.336. Cost per task in USD: 0.61029. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":86.20689655172414,"measurements":[{"observationId":"vals-skillsbench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":61.552,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.20689655172414,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 661.978. Standardfehler: 4.76. Kosten je Aufgabe in USD: 4.516559. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 661.978. Standard error: 4.76. Cost per task in USD: 4.516559. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":97.36842105263158,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":74.899,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.36842105263158,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2882.401. Standardfehler: 1.128. Kosten je Aufgabe in USD: 7.481573. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2882.401. Standard error: 1.128. Cost per task in USD: 7.481573. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":75,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":22.901,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7375.578. Standardfehler: 2.601. Kosten je Aufgabe in USD: 32.231825. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7375.578. Standard error: 2.601. Cost per task in USD: 32.231825. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"vals-swebench-multimodal:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":61.25,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.3076923076923,"participants":14,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 912.134. Standardfehler: 2.226. Kosten je Aufgabe in USD: 5.463719. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 912.134. Standard error: 2.226. Cost per task in USD: 5.463719. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":49.194,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1714.017. Standardfehler: 3.606. Kosten je Aufgabe in USD: 13.180073. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1714.017. Standard error: 3.606. Cost per task in USD: 13.180073. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":94.73684210526316,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.961,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 85.564. Standardfehler: 0.83. Kosten je Aufgabe in USD: 0.346549. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 85.564. Standard error: 0.83. Cost per task in USD: 0.346549. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"vals-vcb-1-100:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":28,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":22,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 10359.295. Standardfehler: 4.488. Kosten je Aufgabe in USD: 149.664612. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 10359.295. Standard error: 4.488. Cost per task in USD: 149.664612. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":94.44444444444444,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-fable-5-1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":90.263,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.44444444444444,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-fable-5-1. Name des Quellmodells: Claude Fable 5.1. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3459.994. Standardfehler: 1.566. Kosten je Aufgabe in USD: 33.367507. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-fable-5-1. Source model name: Claude Fable 5.1. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3459.994. Standard error: 1.566. Cost per task in USD: 33.367507. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":35,"totalFamilies":71,"sourceCount":3,"coverage":0.47497165532879826,"lowerBound":40.65737103746951,"upperBound":93.16020550458968,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":91.55564011280919,"coveredFamilies":11,"totalFamilies":15,"sourceCount":3,"coverage":0.688888888888889,"lowerBound":63.07166318882411,"upperBound":94.1827742999352,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":6.484962406015037,"upperBound":99.3421052631579,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":19.48717948717949,"upperBound":99.48717948717949,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":33.333333333333336,"upperBound":100,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":2,"coverage":0.25,"lowerBound":22.569246391841673,"upperBound":97.56924639184167,"provisional":true,"configurationsDiffer":true},"general":{"score":83.88888888888889,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":50.33333333333333,"upperBound":90.33333333333331,"provisional":true,"configurationsDiffer":true},"finance":{"score":97.22941970310393,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":69.44958550221709,"upperBound":98.02101407364567,"provisional":true,"configurationsDiffer":true},"legal":{"score":89.4588062211013,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":67.09410466582598,"upperBound":92.09410466582598,"provisional":true,"configurationsDiffer":true},"health":{"score":94.78709394205445,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":63.19139596136963,"upperBound":96.52472929470296,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":63.63636363636363,"upperBound":63.63636363636363,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":30.833333333333332,"upperBound":93.33333333333333,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":48.68421052631579,"upperBound":98.68421052631578,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":31.034482758620687,"upperBound":81.03448275862068,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-haiku-4.5","contributions":[{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":36.486486486486484,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-haiku-4-5-20251001","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.295,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.486486486486484,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-4-5-20251001. Name des Quellmodells: Claude Haiku 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-haiku-4-5-20251001. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 7.951. Standardfehler: 0.964. Kosten je Aufgabe in USD: 0.075779. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-4-5-20251001. Source model name: Claude Haiku 4.5. Provider reported by source: Anthropic. Model snapshot: claude-haiku-4-5-20251001. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 7.951. Standard error: 0.964. Cost per task in USD: 0.075779. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":13.157894736842104,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-haiku-4-5-20251001","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":54.263,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.157894736842104,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-4-5-20251001. Name des Quellmodells: Claude Haiku 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-haiku-4-5-20251001. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 177.657. Standardfehler: 1.296. Kosten je Aufgabe in USD: 0.224512. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-4-5-20251001. Source model name: Claude Haiku 4.5. Provider reported by source: Anthropic. Model snapshot: claude-haiku-4-5-20251001. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 177.657. Standard error: 1.296. Cost per task in USD: 0.224512. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":2,"totalFamilies":71,"sourceCount":1,"coverage":0.04591836734693878,"lowerBound":0.8422358986268761,"upperBound":96.25039916393301,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":5.212355212355212,"upperBound":90.92664092664093,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":6.578947368421052,"upperBound":56.578947368421055,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-haiku-5-5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":61.36363636363637,"measurements":[{"observationId":"research:claude-haiku-5-5:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9888888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.36363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-haiku-5-5_xhigh. Epoch-Modellbezeichnung: Claude Haiku 5.5. Anzeigenamenfeld: Claude Haiku 5.5 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: iiEPrSjWXHH6Zq9CK4Zmcr. Startzeit: 2026-10-07T21:16:36.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9888888888888889 und stderr=0.005362818328685681. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: claude-haiku-5-5_xhigh; Epoch model label: Claude Haiku 5.5; Display label field: Claude Haiku 5.5 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: iiEPrSjWXHH6Zq9CK4Zmcr. Started at 2026-10-07T21:16:36.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9888888888888889 and stderr=0.005362818328685681. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":88.15789473684211,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":53.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.15789473684211,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 53,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 53.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1576.85,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":82.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 82,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 82.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":35.411015132783625,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 35,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 35.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":41.8067,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.333333333333336,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":20.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 20,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 20.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":89.02439024390245,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1618.33,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":10.666666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"research:claude-haiku-5-5:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.55,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":66.66666666666667,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-haiku-5-5_max. Epoch-Modellbezeichnung: Claude Haiku 5.5. Anzeigenamenfeld: Claude Haiku 5.5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: KEyuvehfRGtcdgNfNavy62. Startzeit: 2026-10-13T12:40:01.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.55 und stderr=0.049999999999999996. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-haiku-5-5_max; Epoch model label: Claude Haiku 5.5; Display label field: Claude Haiku 5.5 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: KEyuvehfRGtcdgNfNavy62. Started at 2026-10-13T12:40:01.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.55 and stderr=0.049999999999999996. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"research:claude-haiku-5-5:5","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.238,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-haiku-5-5_max. Epoch-Modellbezeichnung: Claude Haiku 5.5. Anzeigenamenfeld: Claude Haiku 5.5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: NwnrUMtveqHDYB2Um4ycmA. Startzeit: 2026-10-09T14:36:06.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.238 und stderr=0.013473586661967093. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-haiku-5-5_max; Epoch model label: Claude Haiku 5.5; Display label field: Claude Haiku 5.5 (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: NwnrUMtveqHDYB2Um4ycmA. Started at 2026-10-09T14:36:06.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.238 and stderr=0.013473586661967093. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":0.5,"score":59.09090909090909,"measurements":[{"observationId":"research:claude-haiku-5-5:0","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.7508771929824561,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":59.09090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-haiku-5-5_max. Epoch-Modellbezeichnung: Claude Haiku 5.5. Anzeigenamenfeld: Claude Haiku 5.5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: 2QAt4ptgGkLtcbVL3hdAvL. Startzeit: 2026-10-07T20:42:31.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.7508771929824561 und stderr=0.02566445410232946. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-haiku-5-5_max; Epoch model label: Claude Haiku 5.5; Display label field: Claude Haiku 5.5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: 2QAt4ptgGkLtcbVL3hdAvL. Started at 2026-10-07T20:42:31.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.7508771929824561 and stderr=0.02566445410232946. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":21.428571428571427,"measurements":[{"observationId":"research:claude-haiku-5-5:2","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.8964646464646465,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":21.428571428571427,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-haiku-5-5_max. Epoch-Modellbezeichnung: Claude Haiku 5.5. Anzeigenamenfeld: Claude Haiku 5.5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: ZEKWTGJiWFscHimMkiUmec. Startzeit: 2026-10-09T14:35:53.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.8964646464646465 und stderr=0.019774484382483127. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-haiku-5-5_max; Epoch model label: Claude Haiku 5.5; Display label field: Claude Haiku 5.5 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: ZEKWTGJiWFscHimMkiUmec. Started at 2026-10-09T14:35:53.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.8964646464646465 and stderr=0.019774484382483127. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":44.3929564411492,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 44,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 44.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":79.78328938694791,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":32.8282828282828,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.7560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 32,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 32.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":36.364,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.86486486486487,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3116.385. Standardfehler: 2.314. Kosten je Aufgabe in USD: 3.117464. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Maximum output tokens: 128000. Latency in seconds: 3116.385. Standard error: 2.314. Cost per task in USD: 3.117464. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":64.37587657784012,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:7d064a11-6e78-4030-b535-cdf940615041","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":20,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.91304347826087,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 20,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/7d064a11-6e78-4030-b535-cdf940615041/claude-haiku-5-5/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Haiku 5.5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 20.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Haiku 5.5 (Max) (Release claude-haiku-5-5, Zeile claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Haiku 5.5 (Max) (release claude-haiku-5-5, row claude-haiku-5-5, UUID 7d064a11-6e78-4030-b535-cdf940615041, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":10,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.83870967741935,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5637.472. Standardfehler: 3.612. Kosten je Aufgabe in USD: 4.523698. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Maximum output tokens: 128000. Latency in seconds: 5637.472. Standard error: 3.612. Cost per task in USD: 4.523698. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":47.36842105263158,"measurements":[{"observationId":"vals-biomysterybench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":70.37,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.36842105263158,"participants":20,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 752.457. Standardfehler: 1.335. Kosten je Aufgabe in USD: 0.512756. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Compute: max. Maximum output tokens: 128000. Latency in seconds: 752.457. Standard error: 1.335. Cost per task in USD: 0.512756. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":72.58064516129032,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.568,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.58064516129032,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4971.306. Standardfehler: 4.231. Kosten je Aufgabe in USD: 6.23191. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 4971.306. Standard error: 4.231. Cost per task in USD: 6.23191. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":87.87878787878788,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":75.476,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.87878787878788,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1959.181. Standardfehler: 5.205. Kosten je Aufgabe in USD: 1.248102. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 1959.181. Standard error: 5.205. Cost per task in USD: 1.248102. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":49.152542372881356,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":59.015,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.152542372881356,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1082.902. Standardfehler: 4.17. Kosten je Aufgabe in USD: 1.146892. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 1082.902. Standard error: 4.17. Cost per task in USD: 1.146892. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":71.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":54.138,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1849. Standardfehler: 2.132. Kosten je Aufgabe in USD: 1.636603. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 1849. Standard error: 2.132. Cost per task in USD: 1.636603. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":30.303030303030305,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":47.278,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.303030303030305,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 753.816. Standardfehler: 10.721. Kosten je Aufgabe in USD: 0.826826. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Maximum output tokens: 128000. Latency in seconds: 753.816. Standard error: 10.721. Cost per task in USD: 0.826826. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":31.147540983606557,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.25,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.147540983606557,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2113.923. Standardfehler: 0.716. Kosten je Aufgabe in USD: 3.394654. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 2113.923. Standard error: 0.716. Cost per task in USD: 3.394654. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":75,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":43.269,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4911.734. Standardfehler: 3.444. Kosten je Aufgabe in USD: 1.633459. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 4911.734. Standard error: 3.444. Cost per task in USD: 1.633459. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":83.58208955223881,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":51.542,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.58208955223881,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 226.464. Standardfehler: 2.166. Kosten je Aufgabe in USD: 0.031937. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Maximum output tokens: 128000. Latency in seconds: 226.464. Standard error: 2.166. Cost per task in USD: 0.031937. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":94.11764705882354,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":89.542,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.11764705882354,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 323.396. Standardfehler: 1.917. Kosten je Aufgabe in USD: 0.043777. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Maximum output tokens: 128000. Latency in seconds: 323.396. Standard error: 1.917. Cost per task in USD: 0.043777. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":62.5,"measurements":[{"observationId":"vals-mysterymechanism:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":35.135,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":21,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 805.031. Standardfehler: 3.211. Kosten je Aufgabe in USD: 0.541024. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 805.031. Standard error: 3.211. Cost per task in USD: 0.541024. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":87.75510204081633,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":2.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.75510204081633,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4389.65. Standardfehler: 1.107. Kosten je Aufgabe in USD: 6.828942. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 4389.65. Standard error: 1.107. Cost per task in USD: 6.828942. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":79.48717948717949,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":89,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.48717948717949,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 418.047. Standardfehler: 3.145. Kosten je Aufgabe in USD: 0.173874. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 418.047. Standard error: 3.145. Cost per task in USD: 0.173874. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":46.55172413793103,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":45.783,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.55172413793103,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 143.279. Standardfehler: 3.423. Kosten je Aufgabe in USD: 0.019327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Maximum output tokens: 128000. Latency in seconds: 143.279. Standard error: 3.423. Cost per task in USD: 0.019327. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":52.63157894736842,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":63.194,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.63157894736842,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2192.484. Standardfehler: 1.254. Kosten je Aufgabe in USD: 0.525402. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 2192.484. Standard error: 1.254. Cost per task in USD: 0.525402. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":50,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":5.725,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3475.503. Standardfehler: 1.438. Kosten je Aufgabe in USD: 3.140034. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Maximum output tokens: 128000. Latency in seconds: 3475.503. Standard error: 1.438. Cost per task in USD: 3.140034. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"vals-swebench-multimodal:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":43.958,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.84615384615385,"participants":14,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 975.904. Standardfehler: 2.268. Kosten je Aufgabe in USD: 0.704848. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Maximum output tokens: 128000. Latency in seconds: 975.904. Standard error: 2.268. Cost per task in USD: 0.704848. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":74.54545454545455,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":31.922,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.54545454545455,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2973.647. Standardfehler: 3.352. Kosten je Aufgabe in USD: 1.164207. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Maximum output tokens: 128000. Latency in seconds: 2973.647. Standard error: 3.352. Cost per task in USD: 1.164207. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-vcb-1-100:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":20.406,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":22,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 14744.334. Standardfehler: 3.792. Kosten je Aufgabe in USD: 8.304533. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Compute: max. Maximum output tokens: 128000. Latency in seconds: 14744.334. Standard error: 3.792. Cost per task in USD: 8.304533. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":98.61111111111111,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-haiku-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":90.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.61111111111111,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-haiku-5-5. Name des Quellmodells: Claude Haiku 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2075.723. Standardfehler: 1.725. Kosten je Aufgabe in USD: 6.071856. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-haiku-5-5. Source model name: Claude Haiku 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Maximum output tokens: 128000. Latency in seconds: 2075.723. Standard error: 1.725. Cost per task in USD: 6.071856. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":36,"totalFamilies":71,"sourceCount":3,"coverage":0.4888888888888888,"lowerBound":31.884410089438042,"upperBound":82.99552120054916,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":66.99932061133163,"coveredFamilies":9,"totalFamilies":15,"sourceCount":2,"coverage":0.5777777777777778,"lowerBound":38.710718575436054,"upperBound":80.93294079765826,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":70.41231126596979,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":40.23560643769703,"upperBound":83.09274929483989,"provisional":true,"configurationsDiffer":true},"math":{"score":68.15850815850816,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.5,"lowerBound":34.07925407925408,"upperBound":84.07925407925407,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":34.146341463414636,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":22.764227642276424,"upperBound":56.09756097560976,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":27.468442447582373,"upperBound":94.13510911424905,"provisional":true,"configurationsDiffer":true},"general":{"score":53.28947368421053,"coveredFamilies":3,"totalFamilies":5,"sourceCount":1,"coverage":0.6000000000000001,"lowerBound":31.97368421052632,"upperBound":71.97368421052632,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":27.909237655000368,"upperBound":85.0520947978575,"provisional":true,"configurationsDiffer":true},"legal":{"score":53.07377049180327,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":26.536885245901637,"upperBound":76.53688524590163,"provisional":true,"configurationsDiffer":true},"health":{"score":88.84986830553117,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":59.23324553702078,"upperBound":92.5665788703541,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":87.87878787878788,"upperBound":87.87878787878788,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":26.31578947368421,"upperBound":76.3157894736842,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":23.275862068965516,"upperBound":73.27586206896552,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-mythos-5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"claude-mythos-5.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"claude-opus-4.5","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":34.21052631578947,"measurements":[{"observationId":"vals-gpqa:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":79.546,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.21052631578947,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 13.54. Standardfehler: 2.463. Kosten je Aufgabe in USD: 0.074932. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 13.54. Standard error: 2.463. Cost per task in USD: 0.074932. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":62.5,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-claude-opus-4.5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":68.5,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":62.5,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/claude-opus-4.5. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-c9e8d3dda73e. Prüfsumme der Quelldaten: c9e8d3dda73e7db4cf1f7f27fd17b5b9cf99cd306a2010121415045b692de2d0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/claude-opus-4.5. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-c9e8d3dda73e. Source payload hash: c9e8d3dda73e7db4cf1f7f27fd17b5b9cf99cd306a2010121415045b692de2d0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":68.75,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-claude-opus-4.5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":80.8,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":68.75,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/claude-opus-4.5. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-53f975fd9591. Prüfsumme der Quelldaten: 53f975fd9591010b73e67024416939079978d0e6f00784a43cdcccc582a69b42. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/claude-opus-4.5. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-53f975fd9591. Source payload hash: 53f975fd9591010b73e67024416939079978d0e6f00784a43cdcccc582a69b42. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":36.170212765957444,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":81.098,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.170212765957444,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 19.557. Standardfehler: 0.941. Kosten je Aufgabe in USD: 0.021549. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 19.557. Standard error: 0.941. Cost per task in USD: 0.021549. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":42.857142857142854,"measurements":[{"observationId":"curated:scale-multichallenge-claude-opus-4.5-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":58.97,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/claude-opus-4.5. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 58,97 % ± 0,42. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-4e857b692726. Prüfsumme der Quelldaten: 4e857b6927268d0e9d6052e7141b07db32abaec5cfdaf89e711cc5b679d713c5. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/claude-opus-4.5. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 58.97% ± 0.42. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-4e857b692726. Source payload hash: 4e857b6927268d0e9d6052e7141b07db32abaec5cfdaf89e711cc5b679d713c5. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"curated:scale-prbench-finance-claude-opus-4.5-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":46.16,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":53.333333333333336,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-opus-4.5. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 46,16 % ± 0,27. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-a16de26d26f0. Prüfsumme der Quelldaten: a16de26d26f0094d97aac5c34552f1c883607ca884e02616d20ae7946cba2d0f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-opus-4.5. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 46.16% ± 0.27. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-a16de26d26f0. Source payload hash: a16de26d26f0094d97aac5c34552f1c883607ca884e02616d20ae7946cba2d0f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"curated:scale-prbench-legal-claude-opus-4.5-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":44.21,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":46.666666666666664,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-opus-4.5. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 44,21 % ± 0,34. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-96a0d2c8de30. Prüfsumme der Quelldaten: 96a0d2c8de30a191eb49d9096b4ad05f763feaf9ad67383630c522c947713da3. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-opus-4.5. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 44.21% ± 0.34. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-96a0d2c8de30. Source payload hash: 96a0d2c8de30a191eb49d9096b4ad05f763feaf9ad67383630c522c947713da3. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":79.76190476190476,"measurements":[{"observationId":"vals-terminal-bench-2:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":58.427,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.76190476190476,"participants":43,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 627.493. Standardfehler: 5.254. Kosten je Aufgabe in USD: 1.175583. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 627.493. Standard error: 5.254. Cost per task in USD: 1.175583. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":51.351351351351354,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.305,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.351351351351354,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 9.401. Standardfehler: 0.96. Kosten je Aufgabe in USD: 0.150159. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 9.401. Standard error: 0.96. Cost per task in USD: 0.150159. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":75.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 13.599. Standardfehler: 1.12. Kosten je Aufgabe in USD: 0.077323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 13.599. Standard error: 1.12. Cost per task in USD: 0.077323. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":44.87179487179487,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.837,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.87179487179487,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 2.376. Standardfehler: 0.434. Kosten je Aufgabe in USD: 0.01319. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 2.376. Standard error: 0.434. Cost per task in USD: 0.01319. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":58.208955223880594,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":45.174,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.208955223880594,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 5.023. Standardfehler: 1.888. Kosten je Aufgabe in USD: 0.006826. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 5.023. Standard error: 1.888. Cost per task in USD: 0.006826. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":64.1025641025641,"measurements":[{"observationId":"vals-medqa:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":93.158,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.1025641025641,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 10.357. Standardfehler: 0.23. Kosten je Aufgabe in USD: 0.043201. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 10.357. Standard error: 0.23. Cost per task in USD: 0.043201. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":51.470588235294116,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.246,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.470588235294116,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 43.305. Standardfehler: 1.926. Kosten je Aufgabe in USD: 0.281674. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 43.305. Standard error: 1.926. Cost per task in USD: 0.281674. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"vals-mgsm:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":94.764,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 5.559. Standardfehler: 1.35. Kosten je Aufgabe in USD: 0.027118. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 5.559. Standard error: 1.35. Cost per task in USD: 0.027118. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":48.05194805194805,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.59,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.05194805194805,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 8.248. Standardfehler: 0.367. Kosten je Aufgabe in USD: 0.052065. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 8.248. Standard error: 0.367. Cost per task in USD: 0.052065. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":79.8076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.68,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.8076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 10.534. Standardfehler: 0.908. Kosten je Aufgabe in USD: 0.092448. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 10.534. Standard error: 0.908. Cost per task in USD: 0.092448. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"vals-poker-agent:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1033.3789020943957,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":9,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 8.852645959989609. Standardfehler: 0. Kosten je Aufgabe in USD: 0.04247890808080806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 8.852645959989609. Standard error: 0. Cost per task in USD: 0.04247890808080806. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":43.10344827586207,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":45.002,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.10344827586207,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 22.727. Standardfehler: 3.352. Kosten je Aufgabe in USD: 0.018815. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 22.727. Standard error: 3.352. Cost per task in USD: 0.018815. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":65.78947368421052,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-opus-4-5-20251101","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.325,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.78947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-5-20251101. Name des Quellmodells: Claude Opus 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-5-20251101. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 14.876. Standardfehler: 0.854. Kosten je Aufgabe in USD: 0.058665. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-5-20251101. Source model name: Claude Opus 4.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-5-20251101. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: high. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 14.876. Standard error: 0.854. Cost per task in USD: 0.058665. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Compute: high. Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":20,"totalFamilies":71,"sourceCount":3,"coverage":0.29492630385487534,"lowerBound":15.738401695837746,"upperBound":86.24577131035022,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":1,"coverage":0.0777777777777778,"lowerBound":2.7044252044252053,"upperBound":94.92664742664742,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":2.443609022556391,"upperBound":95.30075187969925,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":20,"upperBound":100,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.017316017316016,"upperBound":82.68398268398269,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":2,"coverage":0.25,"lowerBound":15.104166666666668,"upperBound":90.10416666666667,"provisional":true,"configurationsDiffer":true},"finance":{"score":62.570462669146885,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":35.754550096655365,"upperBound":78.61169295379823,"provisional":true,"configurationsDiffer":true},"legal":{"score":45.769230769230774,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.5,"lowerBound":22.884615384615387,"upperBound":72.88461538461539,"provisional":true,"configurationsDiffer":true},"health":{"score":57.92736918724627,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":57.92736918724627,"upperBound":57.92736918724627,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":4.5212765957446805,"upperBound":92.02127659574468,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":21.428571428571427,"upperBound":71.42857142857143,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":21.551724137931036,"upperBound":71.55172413793103,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-opus-4.6","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:53c98840-47af-49aa-94e6-469fb17e9a1b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":33.03834808259587,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.23076923076923,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.6 (Max) (Release claude-opus-4-6, Zeile claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/53c98840-47af-49aa-94e6-469fb17e9a1b/claude-opus-4-6/claude-opus-4-6-adaptive. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 33,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.6 (Max) (release claude-opus-4-6, row claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/53c98840-47af-49aa-94e6-469fb17e9a1b/claude-opus-4-6/claude-opus-4-6-adaptive. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 33.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.6 (Max) (Release claude-opus-4-6, Zeile claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.6 (Max) (release claude-opus-4-6, row claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.6 (Max) (Release claude-opus-4-6, Zeile claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.6 (Max) (release claude-opus-4-6, row claude-opus-4-6-adaptive, UUID 53c98840-47af-49aa-94e6-469fb17e9a1b, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":33.333333333333336,"measurements":[{"observationId":"curated:hle-scale-public-claude-opus-4.6","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":34.44,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-6-thinking-max. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 34,44 % ± 1,86. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-9e661b4b9e47. Prüfsumme der Quelldaten: 9e661b4b9e47fc78acc41dd3d83ae7363672deeb05d38a32fd0dcc88c7502935. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-6-thinking-max. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 34.44% ± 1.86. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-9e661b4b9e47. Source payload hash: 9e661b4b9e47fc78acc41dd3d83ae7363672deeb05d38a32fd0dcc88c7502935. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"curated:scale-multichallenge-claude-opus-4.6-non-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":56.02,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: none. Angezeigter Wert: 56,02 % ± 1,08. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-c7bc8d4b82b8. Prüfsumme der Quelldaten: c7bc8d4b82b8a93b33209cb8b5c8da53dc01bac7aa0cd5ca76c4ae365619525d. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: none. Score display: 56.02% ± 1.08. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-c7bc8d4b82b8. Source payload hash: c7bc8d4b82b8a93b33209cb8b5c8da53dc01bac7aa0cd5ca76c4ae365619525d. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: none","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: none"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"none","weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"curated:scale-multinrc-claude-opus-4.6-thinking-max","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":57.06,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 57,06 % ± 2,99. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-4b514d7b2dd5. Prüfsumme der Quelldaten: 4b514d7b2dd55e94cfbbfe497bcdf16bd4bfbf9ab1d083199e10434ca8405d40. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 57.06% ± 2.99. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-4b514d7b2dd5. Source payload hash: 4b514d7b2dd55e94cfbbfe497bcdf16bd4bfbf9ab1d083199e10434ca8405d40. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1. Reasoning-Stufe: max","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":93.33333333333333,"measurements":[{"observationId":"curated:scale-prbench-finance-claude-opus-4.6-non-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":53.28,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":93.33333333333333,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: none. Angezeigter Wert: 53,28 % ± 0,18. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-1196d0ba7a4c. Prüfsumme der Quelldaten: 1196d0ba7a4c8d626776c467c1d9d53e493e848fbce8f12f327c3864f83ac2c4. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: none. Score display: 53.28% ± 0.18. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-1196d0ba7a4c. Source payload hash: 1196d0ba7a4c8d626776c467c1d9d53e493e848fbce8f12f327c3864f83ac2c4. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: none","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: none"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"none","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":93.33333333333333,"measurements":[{"observationId":"curated:scale-prbench-legal-claude-opus-4.6-non-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":52.27,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":93.33333333333333,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: none. Angezeigter Wert: 52,27 % ± 0,66. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-39dd22178287. Prüfsumme der Quelldaten: 39dd221782879e08cf3582225a60f6e91432e1e8d4968e2fb60b53c917fafe6a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: none. Score display: 52.27% ± 0.66. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-39dd22178287. Source payload hash: 39dd221782879e08cf3582225a60f6e91432e1e8d4968e2fb60b53c917fafe6a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: none","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: none"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"none","weight":1,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"curated:scale-tutorbench-claude-opus-4.6-thinking-max","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":53.68,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":55.55555555555556,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 53,68 % ± 1,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-241b6d5fa6d9. Prüfsumme der Quelldaten: 241b6d5fa6d9d25fce4783a0c19c3f18ecedf9c5c98a382e43dc6327a3b55b2c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 53.68% ± 1.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-241b6d5fa6d9. Source payload hash: 241b6d5fa6d9d25fce4783a0c19c3f18ecedf9c5c98a382e43dc6327a3b55b2c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1. Reasoning-Stufe: max","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":68.65671641791045,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":48.244,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.65671641791045,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 4.058. Standardfehler: 2.05. Kosten je Aufgabe in USD: 0.00618. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 4.058. Standard error: 2.05. Cost per task in USD: 0.00618. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":77.94117647058823,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":86.738,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.94117647058823,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 54.318. Standardfehler: 1.942. Kosten je Aufgabe in USD: 0.115121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 54.318. Standard error: 1.942. Cost per task in USD: 0.115121. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":51.388888888888886,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-opus-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":57.573,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.388888888888886,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1278.658. Standardfehler: 4.371. Kosten je Aufgabe in USD: 8.687436. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1278.658. Standard error: 4.371. Cost per task in USD: 8.687436. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"curated:scale-vista-claude-opus-4.6-thinking-max","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":46.07,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":30.76923076923077,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 46,07 % ± 0,26. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-87a83148f3bf. Prüfsumme der Quelldaten: 87a83148f3bfb6d8f82d3dc9e170567d26c6d9e4093248dd3749377527e7b0ba. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 46.07% ± 0.26. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-87a83148f3bf. Source payload hash: 87a83148f3bfb6d8f82d3dc9e170567d26c6d9e4093248dd3749377527e7b0ba. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1. Reasoning-Stufe: max","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"curated:scale-visual-toolbench-claude-opus-4.6-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":27.52,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":77.77777777777777,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/claude-opus-4.6. Name des Quellmodells: Claude Opus 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 27,52 % ± 1,45. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-eb72202a59a1. Prüfsumme der Quelldaten: eb72202a59a1301462dd728b19363526d0f1ed2c2b58120de39b0027aafc74ac. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/claude-opus-4.6. Source model name: Claude Opus 4.6. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 27.52% ± 1.45. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-eb72202a59a1. Source payload hash: eb72202a59a1301462dd728b19363526d0f1ed2c2b58120de39b0027aafc74ac. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":3,"coverage":0.208843537414966,"lowerBound":12.018359727659867,"upperBound":91.13400598616327,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":3.425925925925927,"upperBound":96.75925925925925,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2142857142857143,"lowerBound":11.47186147186147,"upperBound":90.04329004329004,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":5.769230769230768,"upperBound":97.43589743589743,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":13.333333333333332,"upperBound":99.04761904761905,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":23.333333333333332,"upperBound":98.33333333333333,"provisional":true,"configurationsDiffer":true},"health":{"score":73.29894644424934,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":48.865964296166226,"upperBound":82.19929762949957,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":54.27350427350427,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":27.136752136752136,"upperBound":77.13675213675214,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":7.142857142857143,"upperBound":57.142857142857146,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":27.77777777777778,"upperBound":77.77777777777777,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-opus-4.7","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":53.94736842105263,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":43.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.94736842105263,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2/claude-opus-4-7/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.7 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 43,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2/claude-opus-4-7/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.7 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 43.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":74.28571428571429,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":46.6572504708098,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.28571428571429,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2/claude-opus-4-7/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.7 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 46,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2/claude-opus-4-7/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.7 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 46.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.7 (Max) (Release claude-opus-4-7, Zeile claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.7 (Max) (release claude-opus-4-7, row claude-opus-4-7, UUID e9a09db3-8fd6-41dd-ba2f-20e0a2bff7f2, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":73.02631578947368,"measurements":[{"observationId":"vals-gpqa:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":90.152,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.02631578947368,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 202.52. Standardfehler: 1.545. Kosten je Aufgabe in USD: 0.365129. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 202.52. Standard error: 1.545. Cost per task in USD: 0.365129. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":75,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-claude-opus-4.7","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":73.3,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":75,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/claude-opus-4.7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-a7d79c6c1a81. Prüfsumme der Quelldaten: a7d79c6c1a814a1341eb12b4c09af274309b5367920464dde1d5b1bb8d8d87ef. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/claude-opus-4.7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-a7d79c6c1a81. Source payload hash: a7d79c6c1a814a1341eb12b4c09af274309b5367920464dde1d5b1bb8d8d87ef. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-claude-opus-4.7","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":84.6,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/claude-opus-4.7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-62049ade3b48. Prüfsumme der Quelldaten: 62049ade3b484ba643317ee00c6e837652a7189efc368e41874aea7eb708917a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/claude-opus-4.7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-62049ade3b48. Source payload hash: 62049ade3b484ba643317ee00c6e837652a7189efc368e41874aea7eb708917a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":50,"measurements":[{"observationId":"curated:hle-scale-public-claude-opus-4.7","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":36.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":50,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/claude-opus-4.7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 36,20 % ± 1,88. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-f63a678207cc. Prüfsumme der Quelldaten: f63a678207cc25857b90395775e3a58e905919b0e42ab5e1fb5c37b3c4a64205. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/claude-opus-4.7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 36.20% ± 1.88. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-f63a678207cc. Source payload hash: f63a678207cc25857b90395775e3a58e905919b0e42ab5e1fb5c37b3c4a64205. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"lmarena:search:overall:claude-opus-4-7","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1230.1465939891823,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":61.53846153846154,"participants":14,"methodology":{"de":"Arena-Bewertung aus 127.506 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-7. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 127,506 published pairwise votes.. Source model ID: lmarena/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-7. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 127.506 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 127,506 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 127.506 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 127,506 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":61.702127659574465,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":85.549,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.702127659574465,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 70.801. Standardfehler: 0.845. Kosten je Aufgabe in USD: 0.170461. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 70.801. Standard error: 0.845. Cost per task in USD: 0.170461. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":72.88135593220339,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":82,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.88135593220339,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 441.988. Standardfehler: 1.72. Kosten je Aufgabe in USD: 2.423427. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 441.988. Standard error: 1.72. Cost per task in USD: 2.423427. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":75.29761904761905,"measurements":[{"observationId":"vals-terminal-bench-2:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":68.539,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.23809523809524,"participants":43,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 660.89. Standardfehler: 4.95. Kosten je Aufgabe in USD: 0.975907. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 660.89. Standard error: 4.95. Cost per task in USD: 0.975907. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":68.539,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.357142857142854,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 816.587. Standardfehler: 1.297. Kosten je Aufgabe in USD: 1.977087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 816.587. Standard error: 1.297. Cost per task in USD: 1.977087. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":66.12903225806451,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":43.878,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.12903225806451,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4327.01. Standardfehler: 4.22. Kosten je Aufgabe in USD: 36.387681. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4327.01. Standard error: 4.22. Cost per task in USD: 36.387681. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":78.37837837837837,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.084,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.37837837837837,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 17.75. Standardfehler: 0.933. Kosten je Aufgabe in USD: 0.816017. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 17.75. Standard error: 0.933. Cost per task in USD: 0.816017. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":62.71186440677966,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":63.728,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.71186440677966,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2315.595. Standardfehler: 3.106. Kosten je Aufgabe in USD: 12.756352. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2315.595. Standard error: 3.106. Cost per task in USD: 12.756352. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":51.509,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 360.437. Standardfehler: 0.49. Kosten je Aufgabe in USD: 4.032269. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 360.437. Standard error: 0.49. Cost per task in USD: 4.032269. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":67.53246753246754,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.073,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.53246753246754,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 461.91. Standardfehler: 1.006. Kosten je Aufgabe in USD: 0.350587. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 461.91. Standard error: 1.006. Cost per task in USD: 0.350587. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":70.49180327868852,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":6.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.49180327868852,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1483.696. Standardfehler: 1.998. Kosten je Aufgabe in USD: 10.80884. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1483.696. Standard error: 1.998. Cost per task in USD: 10.80884. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":78.2051282051282,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":85.251,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.2051282051282,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 26.622. Standardfehler: 0.451. Kosten je Aufgabe in USD: 0.008332. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 26.622. Standard error: 0.451. Cost per task in USD: 0.008332. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":60,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":38.462,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 511.955. Standardfehler: 3.381. Kosten je Aufgabe in USD: 3.05609. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 511.955. Standard error: 3.381. Cost per task in USD: 3.05609. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":92.53731343283582,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":54.858,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.53731343283582,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 54.251. Standardfehler: 2.205. Kosten je Aufgabe in USD: 0.226314. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 54.251. Standard error: 2.205. Cost per task in USD: 0.226314. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":50,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":82.953,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 67.496. Standardfehler: 1.977. Kosten je Aufgabe in USD: 0.177841. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 67.496. Standard error: 1.977. Cost per task in USD: 0.177841. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":92.20779220779221,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.871,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.20779220779221,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 187.83. Standardfehler: 0.305. Kosten je Aufgabe in USD: 0.061711. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 187.83. Standard error: 0.305. Cost per task in USD: 0.061711. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":96.15384615384616,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":70.27,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.15384615384616,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.101. Standardfehler: 0.896. Kosten je Aufgabe in USD: 0.084417. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 16.101. Standard error: 0.896. Cost per task in USD: 0.084417. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3246.698. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3246.698. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":100,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":56.103,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 124.647. Standardfehler: 3.368. Kosten je Aufgabe in USD: 0.330767. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 124.647. Standard error: 3.368. Cost per task in USD: 0.330767. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":50.90909090909091,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":23.738,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.90909090909091,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 279.035. Standardfehler: 3.063. Kosten je Aufgabe in USD: 1.274806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 279.035. Standard error: 3.063. Cost per task in USD: 1.274806. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":82.89473684210526,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.266,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.89473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 62.071. Standardfehler: 0.841. Kosten je Aufgabe in USD: 0.124592. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 62.071. Standard error: 0.841. Cost per task in USD: 0.124592. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":65.27777777777777,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-opus-4-7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":71.003,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.27777777777777,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-7. Name des Quellmodells: Claude Opus 4.7. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2141.902. Standardfehler: 4.506. Kosten je Aufgabe in USD: 21.407199. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-7. Source model name: Claude Opus 4.7. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2141.902. Standard error: 4.506. Cost per task in USD: 21.407199. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":27,"totalFamilies":71,"sourceCount":5,"coverage":0.3593253968253968,"lowerBound":25.95390498844538,"upperBound":90.0213653059057,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":1,"coverage":0.3222222222222223,"lowerBound":18.997930520132353,"upperBound":86.77570829791013,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":2,"coverage":0.14285714285714285,"lowerBound":8.787593984962406,"upperBound":94.50187969924812,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":30.735930735930737,"upperBound":97.40259740259741,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":2,"coverage":0.3333333333333333,"lowerBound":25.269423558897245,"upperBound":91.93609022556392,"provisional":true,"configurationsDiffer":true},"finance":{"score":70.73020833725563,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":60.62589286050482,"upperBound":74.91160714621911,"provisional":true,"configurationsDiffer":true},"legal":{"score":69.56564382793891,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":52.174232870954185,"upperBound":77.17423287095419,"provisional":true,"configurationsDiffer":true},"health":{"score":71.26865671641791,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":47.51243781094527,"upperBound":80.84577114427861,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":7.712765957446808,"upperBound":95.2127659574468,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":61.53846153846153,"upperBound":61.53846153846153,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":50,"upperBound":100,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-opus-4.8","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":60.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":45,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 45,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 45.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1321.43,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":51.21951219512195,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":77.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.21951219512195,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 77,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 77.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":45.59343691527719,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 45,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 45.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":22.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 22,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 22.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":75.60975609756098,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1453.47,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":46.3768115942029,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":62.244897959183696,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.3768115942029,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 62,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 62.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":89.02439024390245,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":28.75,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"enigma-eval","area":"reasoning","label":{"de":"EnigmaEval","en":"EnigmaEval"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-enigma-eval-2026-07-23-claude-opus-4.8","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/enigma_eval","benchmarkId":"enigma-eval","benchmarkVersion":"open-source release 2026-07-23","comparisonCohort":"scale-enigma-eval-2026-07-23","retrievedAt":"2026-08-26","value":23.51,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":5,"methodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Kennung des Quellmodells: curated/claude-opus-4.8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4.8-xhigh. Evaluator: Scale AI. Datensatztyp: public. Stand der Rangliste: 2026-07-23. Versuche: pass@1. Reasoning-Stufe: xhigh. Angezeigter Wert: 23,51 % ± 2,41. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-5ccace63d82b. Prüfsumme der Quelldaten: 5ccace63d82bb1236aeffd0c9d0760c126999320c83feb7ed8e4170382506715. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Source model ID: curated/claude-opus-4.8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: claude-opus-4.8-xhigh. Evaluator: Scale AI. Dataset type: public. Leaderboard date: 2026-07-23. Attempts: pass@1. Reasoning effort: xhigh. Score display: 23.51% ± 2.41. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-5ccace63d82b. Source payload hash: 5ccace63d82bb1236aeffd0c9d0760c126999320c83feb7ed8e4170382506715. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.","en":"Complex multimodal puzzles with exact answer matching and one attempt."},"scoreRunConfiguration":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Versuche: pass@1. Reasoning-Stufe: xhigh","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Attempts: pass@1. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"scale-enigma-eval-2026-07-23","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":80.26315789473684,"measurements":[{"observationId":"vals-gpqa:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":92.424,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 133.876. Standardfehler: 1.984. Kosten je Aufgabe in USD: 0.339261. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 133.876. Standard error: 1.984. Cost per task in USD: 0.339261. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":48.6561631139944,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 48,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 48.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"lmarena:search:overall:claude-opus-4-8","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1204.1091608362344,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":23.076923076923077,"participants":14,"methodology":{"de":"Arena-Bewertung aus 106.304 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-4-8. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 106,304 published pairwise votes.. Source model ID: lmarena/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: claude-opus-4-8. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 106.304 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 106,304 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 106.304 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 106,304 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":70.2127659574468,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":86.59,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.2127659574468,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 49.044. Standardfehler: 0.819. Kosten je Aufgabe in USD: 0.120663. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 49.044. Standard error: 0.819. Cost per task in USD: 0.120663. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":84.7457627118644,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":88.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.7457627118644,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 566.945. Standardfehler: 1.423. Kosten je Aufgabe in USD: 1.923224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 566.945. Standard error: 1.423. Cost per task in USD: 1.923224. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":75.25726946458656,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:992b7b84-5069-4c6a-9295-834252553d50","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":21.7171717171717,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 21,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/992b7b84-5069-4c6a-9295-834252553d50/claude-opus-4-8/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Model snapshot: Claude Opus 4.8 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 21.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 4.8 (Max) (Release claude-opus-4-8, Zeile claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 4.8 (Max) (release claude-opus-4-8, row claude-opus-4-8, UUID 992b7b84-5069-4c6a-9295-834252553d50, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":70.037,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.61904761904762,"participants":43,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 472.445. Standardfehler: 1.35. Kosten je Aufgabe in USD: 0.883864. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 472.445. Standard error: 1.35. Cost per task in USD: 0.883864. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":71.91,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.714285714285715,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 929.902. Standardfehler: 0.649. Kosten je Aufgabe in USD: 2.409772. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 929.902. Standard error: 0.649. Cost per task in USD: 2.409772. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":23.232,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.4054054054054,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4283.164. Standardfehler: 1.336. Kosten je Aufgabe in USD: 17.140717. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4283.164. Standard error: 1.336. Cost per task in USD: 17.140717. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":27.419354838709676,"measurements":[{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":4.286,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.419354838709676,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6906.538. Standardfehler: 2.438. Kosten je Aufgabe in USD: 23.141733. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6906.538. Standard error: 2.438. Cost per task in USD: 23.141733. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":80.64516129032258,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":47.25,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.64516129032258,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4498.693. Standardfehler: 4.176. Kosten je Aufgabe in USD: 30.514184. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4498.693. Standard error: 4.176. Cost per task in USD: 30.514184. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":83.78378378378379,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.706,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.78378378378379,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.226. Standardfehler: 0.928. Kosten je Aufgabe in USD: 0.855534. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 20.226. Standard error: 0.928. Cost per task in USD: 0.855534. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":79.66101694915254,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":69.373,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.66101694915254,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2632.285. Standardfehler: 2.606. Kosten je Aufgabe in USD: 12.05844. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2632.285. Standard error: 2.606. Cost per task in USD: 12.05844. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":70,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.918,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 547.397. Standardfehler: 0.159. Kosten je Aufgabe in USD: 4.218538. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 547.397. Standard error: 0.159. Cost per task in USD: 4.218538. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":88.31168831168831,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.819,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.31168831168831,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 159.595. Standardfehler: 0.948. Kosten je Aufgabe in USD: 0.327373. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 159.595. Standard error: 0.948. Cost per task in USD: 0.327373. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":80.32786885245902,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":9.583,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.32786885245902,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1447.794. Standardfehler: 2.148. Kosten je Aufgabe in USD: 10.222944. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1447.794. Standard error: 2.148. Cost per task in USD: 10.222944. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.568,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.84615384615385,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3.106. Standardfehler: 0.474. Kosten je Aufgabe in USD: 0.009106. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3.106. Standard error: 0.474. Cost per task in USD: 0.009106. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":76.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":43.75,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 714.036. Standardfehler: 3.448. Kosten je Aufgabe in USD: 2.820942. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 714.036. Standard error: 3.448. Cost per task in USD: 2.820942. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":88.05970149253731,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":53.217,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.05970149253731,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 105.915. Standardfehler: 2.165. Kosten je Aufgabe in USD: 0.350925. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 105.915. Standard error: 2.165. Cost per task in USD: 0.350925. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":73.52941176470588,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":85.755,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.52941176470588,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 82.364. Standardfehler: 1.928. Kosten je Aufgabe in USD: 0.259121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 82.364. Standard error: 1.928. Cost per task in USD: 0.259121. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.585,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 23.44. Standardfehler: 0.303. Kosten je Aufgabe in USD: 0.060716. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 23.44. Standard error: 0.303. Cost per task in USD: 0.060716. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":69.912,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.3076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.814. Standardfehler: 0.893. Kosten je Aufgabe in USD: 0.105627. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 20.814. Standard error: 0.893. Cost per task in USD: 0.105627. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":73.46938775510205,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":1,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.46938775510205,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5523.099. Standardfehler: 0.705. Kosten je Aufgabe in USD: 31.267424. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5523.099. Standard error: 0.705. Cost per task in USD: 31.267424. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":96.55172413793103,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":54.788,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.55172413793103,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 185.594. Standardfehler: 3.347. Kosten je Aufgabe in USD: 0.5055. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 185.594. Standard error: 3.347. Cost per task in USD: 0.5055. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":75.86206896551724,"measurements":[{"observationId":"vals-skillsbench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":59.227,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.86206896551724,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 930.403. Standardfehler: 4.5. Kosten je Aufgabe in USD: 4.766601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 930.403. Standard error: 4.5. Cost per task in USD: 4.766601. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":78.94736842105263,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":68.133,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.94736842105263,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 767.85. Standardfehler: 1.212. Kosten je Aufgabe in USD: 1.889795. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 767.85. Standard error: 1.212. Cost per task in USD: 1.889795. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":29.999,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 479.2. Standardfehler: 3.303. Kosten je Aufgabe in USD: 1.68964. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 479.2. Standard error: 3.303. Cost per task in USD: 1.68964. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":91.44736842105263,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.634,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.44736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 70.33. Standardfehler: 0.838. Kosten je Aufgabe in USD: 0.159139. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 70.33. Standard error: 0.838. Cost per task in USD: 0.159139. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":79.16666666666667,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-opus-4-8","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":82.725,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.16666666666667,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-4-8. Name des Quellmodells: Claude Opus 4.8. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4549.256. Standardfehler: 3.081. Kosten je Aufgabe in USD: 26.879989. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-4-8. Source model name: Claude Opus 4.8. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4549.256. Standard error: 3.081. Cost per task in USD: 26.879989. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":35,"totalFamilies":71,"sourceCount":4,"coverage":0.4752551020408164,"lowerBound":32.49672426795003,"upperBound":84.97121406386839,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":2,"coverage":0.40000000000000013,"lowerBound":30.195515484243554,"upperBound":90.19551548424354,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":55.459724005134774,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.5714285714285714,"lowerBound":31.69127086007701,"upperBound":74.54841371721987,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":89.96674057649668,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":59.97782705099778,"upperBound":93.31116038433112,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":28.946999542590707,"upperBound":87.28033287592403,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":9.27536231884058,"upperBound":89.27536231884058,"provisional":true,"configurationsDiffer":true},"finance":{"score":80.13937084967415,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":68.6908892997207,"upperBound":82.97660358543499,"provisional":true,"configurationsDiffer":true},"legal":{"score":70.28022978842651,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":52.71017234131989,"upperBound":77.71017234131989,"provisional":true,"configurationsDiffer":true},"health":{"score":80.7945566286216,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":53.8630377524144,"upperBound":87.19637108574773,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":8.77659574468085,"upperBound":96.27659574468085,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":23.076923076923077,"upperBound":23.076923076923077,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":39.473684210526315,"upperBound":89.47368421052632,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":48.275862068965516,"upperBound":98.27586206896552,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-opus-5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":61.36363636363637,"measurements":[{"observationId":"research:claude-opus-5:25","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9888888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.36363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-opus-5_max. Epoch-Modellbezeichnung: Claude Opus 5. Anzeigenamenfeld: Claude Opus 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: KxvHAT9SV2dTDn7jBnTk6f. Startzeit: 2026-07-24T18:40:39.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9888888888888889 und stderr=0.011111111111111112. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-opus-5_max; Epoch model label: Claude Opus 5; Display label field: Claude Opus 5 (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: KxvHAT9SV2dTDn7jBnTk6f. Started at 2026-07-24T18:40:39.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9888888888888889 and stderr=0.011111111111111112. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":88.15789473684211,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":53.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.15789473684211,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 53,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 53.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1659.48,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":79.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 79,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 79.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":75.60975609756098,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":56.57355576493251,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 56,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 56.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":21.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 21,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 21.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":97.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1721.15,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":37.06666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":47.61904761904762,"measurements":[{"observationId":"research:claude-opus-5:21","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.43,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":47.61904761904762,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-opus-5_max. Epoch-Modellbezeichnung: Claude Opus 5. Anzeigenamenfeld: Claude Opus 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: RLN98bmQSMnc5Zv39BF4M2. Startzeit: 2026-10-13T13:19:00.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.43 und stderr=0.0497569851956243. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-opus-5_max; Epoch model label: Claude Opus 5; Display label field: Claude Opus 5 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: RLN98bmQSMnc5Zv39BF4M2. Started at 2026-10-13T13:19:00.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.43 and stderr=0.0497569851956243. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":80.3030303030303,"measurements":[{"observationId":"research:claude-opus-5:16","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.7317073170731707,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":83.33333333333333,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-opus-5_max. Epoch-Modellbezeichnung: Claude Opus 5. Anzeigenamenfeld: Claude Opus 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: ji3kMn49Jcr9t4whxcpjj4. Startzeit: 2026-07-24T19:54:20.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.7317073170731707 und stderr=0.07005564203095158. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-opus-5_max; Epoch model label: Claude Opus 5; Display label field: Claude Opus 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: ji3kMn49Jcr9t4whxcpjj4. Started at 2026-07-24T19:54:20.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.7317073170731707 and stderr=0.07005564203095158. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:claude-opus-5:15","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.856140350877193,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":77.27272727272727,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-opus-5_max. Epoch-Modellbezeichnung: Claude Opus 5. Anzeigenamenfeld: Claude Opus 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: ff7GeS7rdperjxSBZ7MHZY. Startzeit: 2026-07-24T19:54:20.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.856140350877193 und stderr=0.020824894575343943. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-opus-5_max; Epoch model label: Claude Opus 5; Display label field: Claude Opus 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: ff7GeS7rdperjxSBZ7MHZY. Started at 2026-07-24T19:54:20.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.856140350877193 and stderr=0.020824894575343943. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":78.7280701754386,"measurements":[{"observationId":"research:claude-opus-5:19","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9387626262626263,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":66.66666666666667,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-opus-5_max. Epoch-Modellbezeichnung: Claude Opus 5. Anzeigenamenfeld: Claude Opus 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: D3mtasTumfvjR4fjmnHWRy. Startzeit: 2026-07-24T18:40:39.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9387626262626263 und stderr=0.014782015056089346. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-opus-5_max; Epoch model label: Claude Opus 5; Display label field: Claude Opus 5 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: D3mtasTumfvjR4fjmnHWRy. Started at 2026-07-24T18:40:39.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9387626262626263 and stderr=0.014782015056089346. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.78947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 39.588. Standardfehler: 1.244. Kosten je Aufgabe in USD: 0.095481. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 39.588. Standard error: 1.244. Cost per task in USD: 0.095481. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":54.8656163113994,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 54,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 54.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":86.11111111111111,"measurements":[{"observationId":"lmarena:webdev:overall:claude-opus-5-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1689.9672356182698,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":86.11111111111111,"participants":37,"methodology":{"de":"Arena-Bewertung aus 17.934 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5-max. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 17,934 published pairwise votes.. Source model ID: lmarena/claude-opus-5-max. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 17.934 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 17,934 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 17.934 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 17,934 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"lmarena:search:overall:claude-opus-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1241.5675643501204,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":84.61538461538461,"participants":14,"methodology":{"de":"Arena-Bewertung aus 13.285 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5-high. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 13,285 published pairwise votes.. Source model ID: lmarena/claude-opus-5-high. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 13.285 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: high","en":"Arena rating from 13,285 published pairwise votes.. Harness: Arena Search. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 13.285 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: high","en":"Arena rating from 13,285 published pairwise votes.. Harness: Arena Search. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"lmarena:overall:claude-opus-5-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1489.5240570151784,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":83.33333333333333,"participants":37,"methodology":{"de":"Arena-Bewertung aus 31.907 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5-max. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 31,907 published pairwise votes.. Source model ID: lmarena/claude-opus-5-max. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 31.907 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 31,907 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 31.907 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 31,907 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":80,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-opus-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1287.4166281489183,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":80,"participants":31,"methodology":{"de":"Arena-Bewertung aus 16.665 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5-high. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 16,665 published pairwise votes.. Source model ID: lmarena/claude-opus-5-high. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 16.665 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 16,665 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 16.665 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 16,665 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":95.36474164133739,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":84.7398843930636,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.85714285714286,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 84,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 84.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":89.884,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.87234042553192,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 30.028. Standardfehler: 0.725. Kosten je Aufgabe in USD: 0.069014. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 30.028. Standard error: 0.725. Cost per task in USD: 0.069014. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":100,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":97,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 576.988. Standardfehler: 0.764. Kosten je Aufgabe in USD: 1.290782. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 576.988. Standard error: 0.764. Cost per task in USD: 1.290782. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":92.40135605989266,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":48.989898989899,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 49,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 49.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":84.644,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.85714285714286,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 752.932. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.886936. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 752.932. Standard error: 0.991. Cost per task in USD: 0.886936. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":53.535,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.78378378378379,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3995.725. Standardfehler: 1.336. Kosten je Aufgabe in USD: 18.596913. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3995.725. Standard error: 1.336. Cost per task in USD: 18.596913. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":80.01402524544179,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:b8fc61f7-5e9a-49e6-8547-6ac56db24627","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":28.571428571428598,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.6086956521739,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 28,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/b8fc61f7-5e9a-49e6-8547-6ac56db24627/claude-opus-5/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 28.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5 (Max) (Release claude-opus-5, Zeile claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Opus 5 (Max) (release claude-opus-5, row claude-opus-5, UUID b8fc61f7-5e9a-49e6-8547-6ac56db24627, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":27.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.41935483870968,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7908.453. Standardfehler: 5.354. Kosten je Aufgabe in USD: 32.543386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7908.453. Standard error: 5.354. Cost per task in USD: 32.543386. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":84.21052631578948,"measurements":[{"observationId":"vals-biomysterybench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":79.259,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.21052631578948,"participants":20,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2195.964. Standardfehler: 2.062. Kosten je Aufgabe in USD: 3.292774. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2195.964. Standard error: 2.062. Cost per task in USD: 3.292774. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":90.3225806451613,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":57.473,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.3225806451613,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 10268.729. Standardfehler: 4.371. Kosten je Aufgabe in USD: 60.509587. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 10268.729. Standard error: 4.371. Cost per task in USD: 60.509587. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":73.194,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Latenz in Sekunden: 28.573. Standardfehler: 0.871. Kosten je Aufgabe in USD: 0.860452. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Latency in seconds: 28.573. Standard error: 0.871. Cost per task in USD: 0.860452. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":42.42424242424242,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":65.357,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.42424242424242,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1165.404. Standardfehler: 5.55. Kosten je Aufgabe in USD: 2.868267. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1165.404. Standard error: 5.55. Cost per task in USD: 2.868267. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":93.22033898305085,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":73.556,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.22033898305085,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1558.673. Standardfehler: 2.24. Kosten je Aufgabe in USD: 6.071566. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1558.673. Standard error: 2.24. Cost per task in USD: 6.071566. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":58.633,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 598.424. Standardfehler: 0.077. Kosten je Aufgabe in USD: 5.124473. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 598.424. Standard error: 0.077. Cost per task in USD: 5.124473. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":84.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3402.045. Standardfehler: 9.964. Kosten je Aufgabe in USD: 16.477241. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3402.045. Standard error: 9.964. Cost per task in USD: 16.477241. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":96.1038961038961,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":89.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.1038961038961,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 59.22. Standardfehler: 0.913. Kosten je Aufgabe in USD: 0.134205. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 59.22. Standard error: 0.913. Cost per task in USD: 0.134205. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":70.49180327868852,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":6.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.49180327868852,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3338.269. Standardfehler: 1.646. Kosten je Aufgabe in USD: 23.669537. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3338.269. Standard error: 1.646. Cost per task in USD: 23.669537. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.974,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.3076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4.415. Standardfehler: 0.417. Kosten je Aufgabe in USD: 0.011976. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4.415. Standard error: 0.417. Cost per task in USD: 0.011976. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":99.16666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":55.288,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":99.16666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1860.719. Standardfehler: 3.456. Kosten je Aufgabe in USD: 6.762498. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1860.719. Standard error: 3.456. Cost per task in USD: 6.762498. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":63.57,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 24.5. Standardfehler: 1.993. Kosten je Aufgabe in USD: 0.156845. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 24.5. Standard error: 1.993. Cost per task in USD: 0.156845. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":95.58823529411765,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":90.985,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.58823529411765,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 76.563. Standardfehler: 1.916. Kosten je Aufgabe in USD: 0.236275. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 76.563. Standard error: 1.916. Cost per task in USD: 0.236275. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":98.7012987012987,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":91.591,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.7012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9.6. Standardfehler: 0.276. Kosten je Aufgabe in USD: 0.027843. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 9.6. Standard error: 0.276. Cost per task in USD: 0.027843. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":72.059,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 7.978. Standardfehler: 0.875. Kosten je Aufgabe in USD: 0.07375. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Latency in seconds: 7.978. Standard error: 0.875. Cost per task in USD: 0.07375. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":75,"measurements":[{"observationId":"vals-mysterymechanism:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":37.387,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":21,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1012.543. Standardfehler: 3.255. Kosten je Aufgabe in USD: 3.280693. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1012.543. Standard error: 3.255. Cost per task in USD: 3.280693. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":90.81632653061224,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":3,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.81632653061224,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 10480.572. Standardfehler: 1.209. Kosten je Aufgabe in USD: 60.289268. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 10480.572. Standard error: 1.209. Cost per task in USD: 60.289268. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":89.74358974358974,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":99,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.74358974358974,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 590.886. Standardfehler: 1. Kosten je Aufgabe in USD: 2.10619. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 590.886. Standard error: 1. Cost per task in USD: 2.10619. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":70.6896551724138,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":49.426,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.6896551724138,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 56.168. Standardfehler: 3.291. Kosten je Aufgabe in USD: 0.192726. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 56.168. Standard error: 3.291. Cost per task in USD: 0.192726. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":79.3103448275862,"measurements":[{"observationId":"vals-skillsbench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":60.441,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.3103448275862,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 505.008. Standardfehler: 4.578. Kosten je Aufgabe in USD: 2.508594. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 505.008. Standard error: 4.578. Cost per task in USD: 2.508594. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":100,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":76.928,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1808.881. Standardfehler: 1.096. Kosten je Aufgabe in USD: 3.949099. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1808.881. Standard error: 1.096. Cost per task in USD: 3.949099. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":70.83333333333333,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":12.214,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.83333333333333,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4858.213. Standardfehler: 2.027. Kosten je Aufgabe in USD: 23.633213. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4858.213. Standard error: 2.027. Cost per task in USD: 23.633213. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":96.36363636363636,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":45.548,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.36363636363636,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1008.306. Standardfehler: 3.591. Kosten je Aufgabe in USD: 5.136554. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1008.306. Standard error: 3.591. Cost per task in USD: 5.136554. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":80.26315789473684,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 58.339. Standardfehler: 0.832. Kosten je Aufgabe in USD: 0.114274. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Latency in seconds: 58.339. Standard error: 0.832. Cost per task in USD: 0.114274. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":90.47619047619048,"measurements":[{"observationId":"vals-vcb-1-100:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":28.533,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.47619047619048,"participants":22,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4905.208. Standardfehler: 4.333. Kosten je Aufgabe in USD: 41.49277. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4905.208. Standard error: 4.333. Cost per task in USD: 41.49277. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":90.27777777777777,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-opus-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":88.404,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.27777777777777,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5. Name des Quellmodells: Claude Opus 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Latenz in Sekunden: 5301.22. Standardfehler: 3.001. Kosten je Aufgabe in USD: 33.876235. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5. Source model name: Claude Opus 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Latency in seconds: 5301.22. Standard error: 3.001. Cost per task in USD: 33.876235. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands","en":"Harness: OpenHands"},"scoreRunConfiguration":{"de":"Harness: OpenHands","en":"Harness: OpenHands"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":81.74448415612066,"coveredFamilies":46,"totalFamilies":71,"sourceCount":4,"coverage":0.68015873015873,"lowerBound":55.599224541107446,"upperBound":87.58335152523445,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":89.24370211695893,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.6222222222222223,"lowerBound":55.52941465055223,"upperBound":93.30719242833,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":70.2180586968777,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":45.14018059084995,"upperBound":80.85446630513567,"provisional":true,"configurationsDiffer":true},"math":{"score":77.13675213675214,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":46.282051282051285,"upperBound":86.28205128205127,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":97.52138105796642,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":65.01425403864428,"upperBound":98.34758737197762,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":36.61828825638529,"upperBound":94.95162158971863,"provisional":true,"configurationsDiffer":true},"general":{"score":78.34429824561404,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":62.675438596491226,"upperBound":82.67543859649122,"provisional":true,"configurationsDiffer":true},"finance":{"score":93.58563331801513,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":80.21625712972724,"upperBound":94.50197141544153,"provisional":true,"configurationsDiffer":true},"legal":{"score":87.32205408434918,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":65.49154056326188,"upperBound":90.49154056326188,"provisional":true,"configurationsDiffer":true},"health":{"score":97.79411764705881,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":65.19607843137254,"upperBound":98.52941176470588,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":42.42424242424242,"upperBound":42.42424242424242,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":87.6823708206687,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":43.84118541033435,"upperBound":93.84118541033435,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":84.61538461538461,"upperBound":84.61538461538461,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":50,"upperBound":100,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":35.3448275862069,"upperBound":85.34482758620689,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-opus-5.5","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":92.10526315789474,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:2f3c4dc9-a450-4303-8697-0237257cf08f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":56.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.10526315789474,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/2f3c4dc9-a450-4303-8697-0237257cf08f/claude-opus-5-5/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 56,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/2f3c4dc9-a450-4303-8697-0237257cf08f/claude-opus-5-5/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 56.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":41.42857142857143,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:2f3c4dc9-a450-4303-8697-0237257cf08f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":38.229755178907695,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.42857142857143,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/2f3c4dc9-a450-4303-8697-0237257cf08f/claude-opus-5-5/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Opus 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 38,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/2f3c4dc9-a450-4303-8697-0237257cf08f/claude-opus-5-5/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Opus 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 38.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Opus 5.5 (Max, Default Fallback) (Release claude-opus-5-5, Zeile claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Opus 5.5 (Max, Default Fallback) (release claude-opus-5-5, row claude-opus-5-5, UUID 2f3c4dc9-a450-4303-8697-0237257cf08f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"lmarena:webdev:overall:claude-opus-5.5-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1811.149195823145,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":100,"participants":37,"methodology":{"de":"Arena-Bewertung aus 2.597 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5.5-max. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5.5-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,597 published pairwise votes.. Source model ID: lmarena/claude-opus-5.5-max. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5.5-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.597 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 2,597 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.597 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 2,597 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"lmarena:search:overall:claude-opus-5.5-medium","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1212.752524435077,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":38.46153846153846,"participants":14,"methodology":{"de":"Arena-Bewertung aus 10.842 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5.5-medium. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5.5-medium. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: medium. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,842 published pairwise votes.. Source model ID: lmarena/claude-opus-5.5-medium. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5.5-medium. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: medium. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.842 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: medium","en":"Arena rating from 10,842 published pairwise votes.. Harness: Arena Search. Reasoning effort: medium"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.842 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: medium","en":"Arena rating from 10,842 published pairwise votes.. Harness: Arena Search. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":97.22222222222223,"measurements":[{"observationId":"lmarena:overall:claude-opus-5.5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1508.082505388002,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":97.22222222222223,"participants":37,"methodology":{"de":"Arena-Bewertung aus 7.687 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5.5-high. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5.5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 7,687 published pairwise votes.. Source model ID: lmarena/claude-opus-5.5-high. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5.5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 7.687 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 7,687 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 7.687 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 7,687 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":90,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-opus-5.5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1292.6884122295935,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":90,"participants":31,"methodology":{"de":"Arena-Bewertung aus 2.508 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-opus-5.5-high. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-opus-5.5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,508 published pairwise votes.. Source model ID: lmarena/claude-opus-5.5-high. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Model snapshot: claude-opus-5.5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.508 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 2,508 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.508 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 2,508 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":99.99999999999999,"measurements":[{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":87.64,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 383.079. Standardfehler: 1.716. Kosten je Aufgabe in USD: 0.496033. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 383.079. Standard error: 1.716. Cost per task in USD: 0.496033. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":65.152,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3810.852. Standardfehler: 0. Kosten je Aufgabe in USD: 13.198632. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3810.852. Standard error: 0. Cost per task in USD: 13.198632. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":91.93548387096774,"measurements":[{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":47.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.93548387096774,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8856.116. Standardfehler: 6.009. Kosten je Aufgabe in USD: 19.121966. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 8856.116. Standard error: 6.009. Cost per task in USD: 19.121966. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":84.21052631578948,"measurements":[{"observationId":"vals-biomysterybench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":79.259,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.21052631578948,"participants":20,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 739.252. Standardfehler: 0.98. Kosten je Aufgabe in USD: 3.642341. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 739.252. Standard error: 0.98. Cost per task in USD: 3.642341. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":96.7741935483871,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":66.646,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.7741935483871,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 11925.532. Standardfehler: 4.329. Kosten je Aufgabe in USD: 112.970791. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 11925.532. Standard error: 4.329. Cost per task in USD: 112.970791. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":24.242424242424242,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":55.357,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.242424242424242,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1091.126. Standardfehler: 5.133. Kosten je Aufgabe in USD: 3.476604. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1091.126. Standard error: 5.133. Cost per task in USD: 3.476604. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":98.30508474576271,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":75.938,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.30508474576271,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2106.434. Standardfehler: 2.383. Kosten je Aufgabe in USD: 11.018974. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2106.434. Standard error: 2.383. Cost per task in USD: 11.018974. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":90,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":58.587,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1987.81. Standardfehler: 0.17. Kosten je Aufgabe in USD: 9.215734. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1987.81. Standard error: 0.17. Cost per task in USD: 9.215734. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":93.93939393939394,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":95.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.93939393939394,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1035.191. Standardfehler: 4.944. Kosten je Aufgabe in USD: 5.258418. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1035.191. Standard error: 4.944. Cost per task in USD: 5.258418. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":55.73770491803279,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":3.75,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.73770491803279,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3175.128. Standardfehler: 1.469. Kosten je Aufgabe in USD: 21.37949. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3175.128. Standard error: 1.469. Cost per task in USD: 21.37949. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":94.16666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":50.481,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.16666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6749.531. Standardfehler: 3.475. Kosten je Aufgabe in USD: 25.163725. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6749.531. Standard error: 3.475. Cost per task in USD: 25.163725. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":79.1044776119403,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.797,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.1044776119403,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 246.876. Standardfehler: 2.273. Kosten je Aufgabe in USD: 0.658237. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 246.876. Standard error: 2.273. Cost per task in USD: 0.658237. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":91.43,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 441.582. Standardfehler: 1.932. Kosten je Aufgabe in USD: 1.154156. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 441.582. Standard error: 1.932. Cost per task in USD: 1.154156. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":95,"measurements":[{"observationId":"vals-mysterymechanism:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":49.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95,"participants":21,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1178.454. Standardfehler: 3.363. Kosten je Aufgabe in USD: 4.619754. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1178.454. Standard error: 3.363. Cost per task in USD: 4.619754. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":18.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8660.211. Standardfehler: 2.753. Kosten je Aufgabe in USD: 68.051144. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 8660.211. Standard error: 2.753. Cost per task in USD: 68.051144. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":97.43589743589743,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":100,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.43589743589743,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 351.035. Standardfehler: 0. Kosten je Aufgabe in USD: 0.961648. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 351.035. Standard error: 0. Cost per task in USD: 0.961648. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":48.275862068965516,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":45.831,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.275862068965516,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 114.201. Standardfehler: 3.36. Kosten je Aufgabe in USD: 0.332. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 114.201. Standard error: 3.36. Cost per task in USD: 0.332. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":92.10526315789474,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":70.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.10526315789474,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4771.81. Standardfehler: 1.185. Kosten je Aufgabe in USD: 5.952407. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4771.81. Standard error: 1.185. Cost per task in USD: 5.952407. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":33.588,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5795.029. Standardfehler: 2.923. Kosten je Aufgabe in USD: 34.654445. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5795.029. Standard error: 2.923. Cost per task in USD: 34.654445. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-swebench-multimodal:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":70.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":14,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1111.602. Standardfehler: 2.09. Kosten je Aufgabe in USD: 4.920591. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1111.602. Standard error: 2.09. Cost per task in USD: 4.920591. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":98.18181818181819,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":45.573,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.18181818181819,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4012.548. Standardfehler: 3.593. Kosten je Aufgabe in USD: 15.086184. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4012.548. Standard error: 3.593. Cost per task in USD: 15.086184. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":95.23809523809524,"measurements":[{"observationId":"vals-vcb-1-100:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":30.361,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.23809523809524,"participants":22,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Rechenaufwand: xhigh. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8909.323. Standardfehler: 4.829. Kosten je Aufgabe in USD: 44.115239. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Compute: xhigh. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 8909.323. Standard error: 4.829. Cost per task in USD: 44.115239. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: xhigh. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: xhigh. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: xhigh. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: xhigh. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":95.83333333333333,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-opus-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":90.294,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.83333333333333,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-opus-5-5. Name des Quellmodells: Claude Opus 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5766.524. Standardfehler: 1.526. Kosten je Aufgabe in USD: 57.922811. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-opus-5-5. Source model name: Claude Opus 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5766.524. Standard error: 1.526. Cost per task in USD: 57.922811. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":28,"totalFamilies":71,"sourceCount":3,"coverage":0.46672335600907033,"lowerBound":32.03376900009258,"upperBound":85.36143339918556,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":97.15992655251405,"coveredFamilies":9,"totalFamilies":15,"sourceCount":2,"coverage":0.5222222222222225,"lowerBound":50.7390727552018,"upperBound":98.51685053297956,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":19.48717948717949,"upperBound":99.48717948717949,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":11.127819548872182,"upperBound":94.46115288220552,"provisional":true,"configurationsDiffer":true},"general":{"score":92.0248538011696,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":73.61988304093569,"upperBound":93.61988304093569,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":40.92670041822584,"upperBound":98.06955756108297,"provisional":true,"configurationsDiffer":true},"legal":{"score":74.95218579234974,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":37.47609289617487,"upperBound":87.47609289617486,"provisional":true,"configurationsDiffer":true},"health":{"score":89.55223880597015,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":59.701492537313435,"upperBound":93.03482587064677,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":24.242424242424242,"upperBound":24.242424242424242,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":22.5,"upperBound":97.5,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":38.46153846153846,"upperBound":38.46153846153846,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":46.05263157894737,"upperBound":96.05263157894737,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":24.137931034482758,"upperBound":74.13793103448276,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-sonnet-4.5","contributions":[{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-multichallenge-claude-sonnet-4.5-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":55.32,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/claude-sonnet-4.5. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 55,32 % ± 1,47. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-93c4c6df84ba. Prüfsumme der Quelldaten: 93c4c6df84baa57ecc302765b454f1d1d402750d6ee8968f2085772a44e192b1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/claude-sonnet-4.5. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 55.32% ± 1.47. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-93c4c6df84ba. Source payload hash: 93c4c6df84baa57ecc302765b454f1d1d402750d6ee8968f2085772a44e192b1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":40,"measurements":[{"observationId":"curated:scale-prbench-finance-claude-sonnet-4.5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":43.79,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":40,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-sonnet-4.5. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 43,79 % ± 0,4. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-db93e34ac02d. Prüfsumme der Quelldaten: db93e34ac02d5628c3ec931b3f6d17257e1b7695c21cf8aee846e136474ca9b9. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-sonnet-4.5. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 43.79% ± 0.4. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-db93e34ac02d. Source payload hash: db93e34ac02d5628c3ec931b3f6d17257e1b7695c21cf8aee846e136474ca9b9. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":20,"measurements":[{"observationId":"curated:scale-prbench-legal-claude-sonnet-4.5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":40.84,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":20,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/claude-sonnet-4.5. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 40,84 % ± 0,4. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-cfd9f8c7d859. Prüfsumme der Quelldaten: cfd9f8c7d85930ec8d7acf7515a8200c29b116b3913b0affc48e4e68b3103115. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/claude-sonnet-4.5. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 40.84% ± 0.4. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-cfd9f8c7d859. Source payload hash: cfd9f8c7d85930ec8d7acf7515a8200c29b116b3913b0affc48e4e68b3103115. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"scicode","area":"coding","label":{"de":"SciCode","en":"SciCode"},"weight":0.004761904761904762,"coverage":1,"score":25,"measurements":[{"observationId":"curated:hal-scicode-scicode-tool-calling-overall-claude-sonnet-4.5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scicode","benchmarkId":"hal-scicode","benchmarkVersion":"65-problem main set","comparisonCohort":"hal-scicode-scicode-tool-calling-overall","retrievedAt":"2026-08-05","value":3.08,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":25,"participants":5,"methodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Kennung des Quellmodells: curated/claude-sonnet-4.5. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Kosten je Aufgabe in USD: 1.6044615384615386. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-c67a4c4ed842. Prüfsumme der Quelldaten: c67a4c4ed8428dfb71febdf30c9d21b46b0644189ef4d4de5ce334f3f2edc12c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Source model ID: curated/claude-sonnet-4.5. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Evaluator: Princeton HAL. Dataset type: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Cost per task in USD: 1.6044615384615386. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-c67a4c4ed842. Source payload hash: c67a4c4ed8428dfb71febdf30c9d21b46b0644189ef4d4de5ce334f3f2edc12c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65."},"scoreRunConfiguration":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling"},"scoreComparisonSetup":{"publicationCohort":"hal-scicode-scicode-tool-calling-overall","datasetType":"public","harness":"SciCode Tool Calling","agent":"SciCode Tool Calling","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":41.21621621621622,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-sonnet-4-5-20250929","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.21621621621622,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-5-20250929. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 13.383. Standardfehler: 0.962. Kosten je Aufgabe in USD: 0.237245. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-5-20250929. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 64000. Latency in seconds: 13.383. Standard error: 0.962. Cost per task in USD: 0.237245. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 64000","en":"Temperature: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":31.34328358208955,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-sonnet-4-5-20250929","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.569,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.34328358208955,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-5-20250929. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 12.006. Standardfehler: 1.995. Kosten je Aufgabe in USD: 0.042403. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-5-20250929. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 12.006. Standard error: 1.995. Cost per task in USD: 0.042403. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":66.17647058823529,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-sonnet-4-5-20250929","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":84.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.17647058823529,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-5-20250929. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 44.419. Standardfehler: 1.929. Kosten je Aufgabe in USD: 0.054649. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-5-20250929. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 44.419. Standard error: 1.929. Cost per task in USD: 0.054649. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":62.5,"measurements":[{"observationId":"vals-poker-agent:overall:anthropic/claude-sonnet-4-5-20250929","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1055.504459218733,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":9,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-5-20250929. Name des Quellmodells: Claude Sonnet 4.5. Modellstand: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 16.67560795007542. Standardfehler: 0. Kosten je Aufgabe in USD: 0.027591309991158296. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-5-20250929. Source model name: Claude Sonnet 4.5. Model snapshot: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 16.67560795007542. Standard error: 0. Cost per task in USD: 0.027591309991158296. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":10.344827586206897,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-sonnet-4-5-20250929","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":32.88,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.344827586206897,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-5-20250929. Name des Quellmodells: Claude Sonnet 4.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.542. Standardfehler: 3.141. Kosten je Aufgabe in USD: 0.046215. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-5-20250929. Source model name: Claude Sonnet 4.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-4-5-20250929. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 16.542. Standard error: 3.141. Cost per task in USD: 0.046215. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":3,"coverage":0.16802721088435377,"lowerBound":4.368308222775833,"upperBound":87.56558713434046,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":1.6666666666666672,"upperBound":95,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":5.208333333333333,"upperBound":96.87499999999999,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":2,"coverage":0.2857142857142857,"lowerBound":11.602316602316602,"upperBound":83.03088803088804,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":5,"upperBound":80,"provisional":true,"configurationsDiffer":true},"health":{"score":48.75987708516242,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":32.50658472344161,"upperBound":65.83991805677495,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":0,"upperBound":50,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":5.172413793103448,"upperBound":55.172413793103445,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-sonnet-4.6","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":22.36842105263158,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":20,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":22.36842105263158,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 20,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 20.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1062.96,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":80,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":28.023598820058996,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.84615384615385,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 28,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 28.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":20.098073664518367,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 20,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 20.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":15.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 15,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 15.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1232,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":45.714285714285715,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":39.8022598870057,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.714285714285715,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 39,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 39.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":12.216666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":55.26315789473684,"measurements":[{"observationId":"vals-gpqa:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":85.606,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 377.864. Standardfehler: 2.406. Kosten je Aufgabe in USD: 0.408666. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 377.864. Standard error: 2.406. Cost per task in USD: 0.408666. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":47.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":33.595922150139,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 33,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 33.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":52.12765957446808,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.584,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.12765957446808,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 131.728. Standardfehler: 0.891. Kosten je Aufgabe in USD: 0.144122. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 131.728. Standard error: 0.891. Cost per task in USD: 0.144122. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":53.389830508474574,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":77.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.389830508474574,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 511.785. Standardfehler: 1.872. Kosten je Aufgabe in USD: 1.303853. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 511.785. Standard error: 1.872. Cost per task in USD: 1.303853. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":59.94337979094078,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:df8d14e0-3997-4e4d-b4ad-9c047acc9c69","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":3.03030303030303,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 3,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/df8d14e0-3997-4e4d-b4ad-9c047acc9c69/claude-sonnet-4-6/claude-sonnet-4-6-adaptive. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 4.6 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 3.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 4.6 (Max) (Release claude-sonnet-4-6, Zeile claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 4.6 (Max) (release claude-sonnet-4-6, row claude-sonnet-4-6-adaptive, UUID df8d14e0-3997-4e4d-b4ad-9c047acc9c69, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":59.551,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":43,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 615.5. Standardfehler: 5.232. Kosten je Aufgabe in USD: 0.518966. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 615.5. Standard error: 5.232. Cost per task in USD: 0.518966. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":57.303,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.392857142857146,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 704.225. Standardfehler: 0.649. Kosten je Aufgabe in USD: 0.572392. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 704.225. Standard error: 0.649. Cost per task in USD: 0.572392. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: false. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":62.903225806451616,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":39.892,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.903225806451616,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5979.033. Standardfehler: 4.138. Kosten je Aufgabe in USD: 38.507642. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5979.033. Standard error: 4.138. Cost per task in USD: 38.507642. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":73.64864864864865,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":65.307,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.64864864864865,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.483. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.307405. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 18.483. Standard error: 0.938. Cost per task in USD: 0.307405. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":54.23728813559322,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":60.154,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.23728813559322,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3255.853. Standardfehler: 3.276. Kosten je Aufgabe in USD: 7.22787. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3255.853. Standard error: 3.276. Cost per task in USD: 7.22787. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":51.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":51.035,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 695.588. Standardfehler: 0.329. Kosten je Aufgabe in USD: 2.409559. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 695.588. Standard error: 0.329. Cost per task in USD: 2.409559. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":48.05194805194805,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.091,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.05194805194805,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 295.009. Standardfehler: 1.063. Kosten je Aufgabe in USD: 0.370575. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 295.009. Standard error: 1.063. Cost per task in USD: 0.370575. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":61.47540983606557,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":5,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.47540983606557,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1122.707. Standardfehler: 1.646. Kosten je Aufgabe in USD: 3.042346. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1122.707. Standard error: 1.646. Cost per task in USD: 3.042346. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":42.30769230769231,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.12,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.30769230769231,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 21.628. Standardfehler: 0.476. Kosten je Aufgabe in USD: 0.004202. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 21.628. Standard error: 0.476. Cost per task in USD: 0.004202. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":60,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":38.462,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 989.604. Standardfehler: 3.381. Kosten je Aufgabe in USD: 2.302948. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 989.604. Standard error: 3.381. Cost per task in USD: 2.302948. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"vals-medqa:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":92.058,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.84615384615385,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning: true. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 35.119. Standardfehler: 0.247. Kosten je Aufgabe in USD: 0.035415. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning: true. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 35.119. Standard error: 0.247. Cost per task in USD: 0.035415. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.341,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.72727272727273,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 55.406. Standardfehler: 0.434. Kosten je Aufgabe in USD: 0.066412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 55.406. Standard error: 0.434. Cost per task in USD: 0.066412. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":67.3076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":67.726,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.3076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 29.74. Standardfehler: 0.921. Kosten je Aufgabe in USD: 0.04467. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 29.74. Standard error: 0.921. Cost per task in USD: 0.04467. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4349.841. Standardfehler: 0.5. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4349.841. Standard error: 0.5. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":53.44827586206897,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":46.582,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.44827586206897,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 163.365. Standardfehler: 3.43. Kosten je Aufgabe in USD: 0.234647. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 163.365. Standard error: 3.43. Cost per task in USD: 0.234647. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":37.93103448275862,"measurements":[{"observationId":"vals-skillsbench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":49.052,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.93103448275862,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 860.178. Standardfehler: 4.64. Kosten je Aufgabe in USD: 1.382228. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 860.178. Standard error: 4.64. Cost per task in USD: 1.382228. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":47.36842105263158,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":62.449,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.36842105263158,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 882.995. Standardfehler: 1.26. Kosten je Aufgabe in USD: 1.105228. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 882.995. Standard error: 1.26. Cost per task in USD: 1.105228. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":58.18181818181818,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":25.84,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.18181818181818,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 707.307. Standardfehler: 3.154. Kosten je Aufgabe in USD: 1.193501. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 707.307. Standard error: 3.154. Cost per task in USD: 1.193501. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":77.106,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 127.879. Standardfehler: 0.818. Kosten je Aufgabe in USD: 0.127973. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 127.879. Standard error: 0.818. Cost per task in USD: 0.127973. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":50,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-sonnet-4-6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":51.476,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-4-6. Name des Quellmodells: Claude Sonnet 4.6. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1572.118. Standardfehler: 4.645. Kosten je Aufgabe in USD: 5.907892. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-4-6. Source model name: Claude Sonnet 4.6. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1572.118. Standard error: 4.645. Cost per task in USD: 5.907892. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning: true. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning: true. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":32,"totalFamilies":71,"sourceCount":2,"coverage":0.36425736961451244,"lowerBound":21.07397098990111,"upperBound":84.64823402844988,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.3555555555555556,"lowerBound":19.853799149182716,"upperBound":84.29824359362716,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":26.334128002934165,"upperBound":83.4769851457913,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":71.72949002217295,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":47.81966001478197,"upperBound":81.1529933481153,"provisional":true,"configurationsDiffer":true},"agentic":{"score":46.87893971752272,"coveredFamilies":7,"totalFamilies":12,"sourceCount":2,"coverage":0.5833333333333334,"lowerBound":27.34604816855492,"upperBound":69.01271483522159,"provisional":true,"configurationsDiffer":true},"finance":{"score":67.50701899006984,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":57.86315913434557,"upperBound":72.14887342005986,"provisional":true,"configurationsDiffer":true},"legal":{"score":54.594367381252624,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":40.94577553593947,"upperBound":65.94577553593948,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":17.94871794871795,"upperBound":84.61538461538461,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":6.51595744680851,"upperBound":94.01595744680851,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":23.68421052631579,"upperBound":73.6842105263158,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":26.724137931034484,"upperBound":76.72413793103448,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-sonnet-5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"research:claude-sonnet-5:14","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.8,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: XRutLYUVUePe69APTSTw8S. Startzeit: 2026-08-06T22:21:48.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.8 und stderr=0.06030226891555273. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: XRutLYUVUePe69APTSTw8S. Started at 2026-08-06T22:21:48.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.8 and stderr=0.06030226891555273. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":67.10526315789474,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":46.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.10526315789474,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 46,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 46.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":75.60975609756098,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1358.4,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":82,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 82,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 82.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":36.51267571703639,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 36,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 36.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":13.200000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 13,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 13.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1463.81,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":74.39024390243902,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":16.45,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.39024390243902,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":54.76190476190476,"measurements":[{"observationId":"research:claude-sonnet-5:10","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.45,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":54.76190476190476,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: PN9exWpeJGXJB8tZGX32e4. Startzeit: 2026-10-13T13:23:26.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.45 und stderr=0.05. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: PN9exWpeJGXJB8tZGX32e4. Started at 2026-10-13T13:23:26.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.45 and stderr=0.05. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":5.555555555555555,"measurements":[{"observationId":"research:claude-sonnet-5:12","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.337,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":5.555555555555555,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: LPo74T9L3CawnKX5LLUrSZ. Startzeit: 2026-08-27T19:27:42.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.337 und stderr=0.014955087918653461. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: LPo74T9L3CawnKX5LLUrSZ. Started at 2026-08-27T19:27:42.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.337 and stderr=0.014955087918653461. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":38.13131313131313,"measurements":[{"observationId":"research:claude-sonnet-5:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.2926829268292683,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":44.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: bqiVinHXFcgyFGPstJwXJN. Startzeit: 2026-06-30T21:54:08.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.2926829268292683 und stderr=0.07194088392074452. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: bqiVinHXFcgyFGPstJwXJN. Started at 2026-06-30T21:54:08.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.2926829268292683 and stderr=0.07194088392074452. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:claude-sonnet-5:6","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.656140350877193,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":31.818181818181817,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: V6YuGpcu8MCbJNXKoJMA6b. Startzeit: 2026-06-30T21:54:08.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.656140350877193 und stderr=0.02818576398907258. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: V6YuGpcu8MCbJNXKoJMA6b. Started at 2026-06-30T21:54:08.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.656140350877193 and stderr=0.02818576398907258. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":36.2625313283208,"measurements":[{"observationId":"research:claude-sonnet-5:8","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.803030303030303,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":4.761904761904762,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: claude-sonnet-5_max. Epoch-Modellbezeichnung: Claude Sonnet 5. Anzeigenamenfeld: Claude Sonnet 5 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: FNXAt76ZUS3sWhQDFMga7U. Startzeit: 2026-08-06T22:21:47.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.803030303030303 und stderr=0.028335609732463334. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: claude-sonnet-5_max; Epoch model label: Claude Sonnet 5; Display label field: Claude Sonnet 5 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: FNXAt76ZUS3sWhQDFMga7U. Started at 2026-08-06T22:21:47.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.803030303030303 and stderr=0.028335609732463334. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":88.889,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.76315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 63.234. Standardfehler: 2.224. Kosten je Aufgabe in USD: 0.077709. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 63.234. Standard error: 2.224. Cost per task in USD: 0.077709. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":41.2882298424467,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":38.888888888888886,"measurements":[{"observationId":"lmarena:webdev:overall:claude-sonnet-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1539.918730362966,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":38.888888888888886,"participants":37,"methodology":{"de":"Arena-Bewertung aus 15.175 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5-high. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 15,175 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5-high. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 15.175 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 15,175 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 15.175 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 15,175 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"lmarena:overall:claude-sonnet-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1461.28354048367,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":33.333333333333336,"participants":37,"methodology":{"de":"Arena-Bewertung aus 48.998 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5-high. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 48,998 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5-high. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 48.998 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 48,998 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 48.998 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 48,998 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-sonnet-5-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1263.2553173601593,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":33.333333333333336,"participants":31,"methodology":{"de":"Arena-Bewertung aus 13.503 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5-high. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 13,503 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5-high. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 13.503 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 13,503 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 13.503 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 13,503 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":44.680851063829785,"measurements":[{"observationId":"vals-mmmu:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.006,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.680851063829785,"participants":48,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.817. Standardfehler: 0.903. Kosten je Aufgabe in USD: 0.033133. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 18.817. Standard error: 0.903. Cost per task in USD: 0.033133. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":66.94915254237289,"measurements":[{"observationId":"vals-swebench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":79.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.94915254237289,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 962.366. Standardfehler: 1.804. Kosten je Aufgabe in USD: 1.489899. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 962.366. Standard error: 1.804. Cost per task in USD: 1.489899. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":63.867713532347686,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:23c86e4a-c769-43c0-a056-79e3cd15834f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":14.1414141414141,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 14,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/23c86e4a-c769-43c0-a056-79e3cd15834f/claude-sonnet-5/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 14.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5 (Max) (Release claude-sonnet-5, Zeile claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Claude Sonnet 5 (Max) (release claude-sonnet-5, row claude-sonnet-5, UUID 23c86e4a-c769-43c0-a056-79e3cd15834f, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":74.532,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.32142857142857,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 635.043. Standardfehler: 2.085. Kosten je Aufgabe in USD: 0.534595. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 635.043. Standard error: 2.085. Cost per task in USD: 0.534595. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":10.606,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.675675675675677,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6297.51. Standardfehler: 1.515. Kosten je Aufgabe in USD: 26.04996. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6297.51. Standard error: 1.515. Cost per task in USD: 26.04996. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":38.70967741935484,"measurements":[{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":5.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.70967741935484,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 11157.924. Standardfehler: 2.794. Kosten je Aufgabe in USD: 30.299862. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 11157.924. Standard error: 2.794. Cost per task in USD: 30.299862. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":69.35483870967742,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.392,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.35483870967742,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7026.544. Standardfehler: 4.246. Kosten je Aufgabe in USD: 35.310438. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7026.544. Standard error: 4.246. Cost per task in USD: 35.310438. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":86.48648648648648,"measurements":[{"observationId":"vals-corp-fin-v2:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.977,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.48648648648648,"participants":75,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.628. Standardfehler: 0.926. Kosten je Aufgabe in USD: 0.516119. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 18.628. Standard error: 0.926. Cost per task in USD: 0.516119. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":61.905,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1017.033. Standardfehler: 5.742. Kosten je Aufgabe in USD: 1.888947. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1017.033. Standard error: 5.742. Cost per task in USD: 1.888947. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":69.49152542372882,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":66.316,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.49152542372882,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2985.704. Standardfehler: 3.006. Kosten je Aufgabe in USD: 10.293794. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2985.704. Standard error: 3.006. Cost per task in USD: 10.293794. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":68.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.909,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 792.086. Standardfehler: 0.518. Kosten je Aufgabe in USD: 0.750759. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 792.086. Standard error: 0.518. Cost per task in USD: 0.750759. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":24.242424242424242,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":45,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.242424242424242,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3583.884. Standardfehler: 2.754. Kosten je Aufgabe in USD: 12.889951. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3583.884. Standard error: 2.754. Cost per task in USD: 12.889951. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":53.246753246753244,"measurements":[{"observationId":"vals-lcb:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.429,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.246753246753244,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 77.021. Standardfehler: 1.088. Kosten je Aufgabe in USD: 0.094655. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 77.021. Standard error: 1.088. Cost per task in USD: 0.094655. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":61.47540983606557,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":5,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.47540983606557,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2331.577. Standardfehler: 1.646. Kosten je Aufgabe in USD: 8.953859. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2331.577. Standard error: 1.646. Cost per task in USD: 8.953859. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":60.256410256410255,"measurements":[{"observationId":"vals-legal-bench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.922,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.256410256410255,"participants":79,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4.901. Standardfehler: 0.458. Kosten je Aufgabe in USD: 0.005759. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4.901. Standard error: 0.458. Cost per task in USD: 0.005759. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":41.827,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1545.997. Standardfehler: 3.428. Kosten je Aufgabe in USD: 2.719446. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1545.997. Standard error: 3.428. Cost per task in USD: 2.719446. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":64.17910447761194,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":47.54,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.17910447761194,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 134.934. Standardfehler: 2.274. Kosten je Aufgabe in USD: 0.278799. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 134.934. Standard error: 2.274. Cost per task in USD: 0.278799. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":25,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":76.054,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 251.703. Standardfehler: 3.05. Kosten je Aufgabe in USD: 0.433684. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 251.703. Standard error: 3.05. Cost per task in USD: 0.433684. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":75.32467532467533,"measurements":[{"observationId":"vals-mmlu-pro:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.545,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.32467532467533,"participants":78,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 25.191. Standardfehler: 0.368. Kosten je Aufgabe in USD: 0.042996. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 25.191. Standard error: 0.368. Cost per task in USD: 0.042996. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":94.23076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":70.032,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.23076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 28.668. Standardfehler: 0.897. Kosten je Aufgabe in USD: 0.080204. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 28.668. Standard error: 0.897. Cost per task in USD: 0.080204. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5471.639. Standardfehler: 0. Kosten je Aufgabe in USD: 24.358317. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5471.639. Standard error: 0. Cost per task in USD: 24.358317. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":77,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 898.079. Standardfehler: 4.23. Kosten je Aufgabe in USD: 1.373684. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 898.079. Standard error: 4.23. Cost per task in USD: 1.373684. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":67.24137931034483,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":48.925,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.24137931034483,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 434.2. Standardfehler: 3.405. Kosten je Aufgabe in USD: 0.588528. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 434.2. Standard error: 3.405. Cost per task in USD: 0.588528. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Compute: max. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":31.03448275862069,"measurements":[{"observationId":"vals-skillsbench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":46.476,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.03448275862069,"participants":30,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 911.674. Standardfehler: 4.491. Kosten je Aufgabe in USD: 2.89733. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 911.674. Standard error: 4.491. Cost per task in USD: 2.89733. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":65.78947368421052,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":66.035,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.78947368421052,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1268.852. Standardfehler: 1.232. Kosten je Aufgabe in USD: 1.290695. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1268.852. Standard error: 1.232. Cost per task in USD: 1.290695. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":12.5,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0.382,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.5,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 13984.551. Standardfehler: 0.382. Kosten je Aufgabe in USD: 18.493954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 13984.551. Standard error: 0.382. Cost per task in USD: 18.493954. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":61.81818181818182,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":29.374,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.81818181818182,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1158.894. Standardfehler: 3.248. Kosten je Aufgabe in USD: 1.794774. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1158.894. Standard error: 3.248. Cost per task in USD: 1.794774. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":91.44736842105263,"measurements":[{"observationId":"vals-tax-eval-v2:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.634,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.44736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 202.488. Standardfehler: 0.836. Kosten je Aufgabe in USD: 0.301619. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 202.488. Standard error: 0.836. Cost per task in USD: 0.301619. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":19.047619047619047,"measurements":[{"observationId":"vals-vcb-1-100:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":13.822,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.047619047619047,"participants":22,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15369.858. Standardfehler: 2.985. Kosten je Aufgabe in USD: 71.148517. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 15369.858. Standard error: 2.985. Cost per task in USD: 71.148517. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":76.38888888888889,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-sonnet-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":81.325,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.38888888888889,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5. Name des Quellmodells: Claude Sonnet 5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4001.424. Standardfehler: 3.046. Kosten je Aufgabe in USD: 25.38542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5. Source model name: Claude Sonnet 5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4001.424. Standard error: 3.046. Cost per task in USD: 25.38542. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":52.258047674547754,"coveredFamilies":44,"totalFamilies":71,"sourceCount":4,"coverage":0.5926587301587303,"lowerBound":30.971188175371857,"upperBound":71.70531515949884,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":46.60887379627615,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.5888888888888891,"lowerBound":27.44744790225152,"upperBound":68.55855901336261,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":59.79404812845121,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":38.439030939718634,"upperBound":74.15331665400436,"provisional":true,"configurationsDiffer":true},"math":{"score":34.93265993265993,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":20.95959595959596,"upperBound":60.95959595959595,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":51.756824927556636,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":51.756824927556636,"upperBound":51.756824927556636,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":26.166117037758397,"upperBound":84.49945037109173,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":9.166666666666668,"upperBound":69.16666666666667,"provisional":true,"configurationsDiffer":true},"finance":{"score":78.63461078559206,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":67.4010949590789,"upperBound":81.6868092447932,"provisional":true,"configurationsDiffer":true},"legal":{"score":65.02171780860306,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":48.766288356452286,"upperBound":73.76628835645229,"provisional":true,"configurationsDiffer":true},"health":{"score":44.58955223880597,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":29.72636815920398,"upperBound":63.059701492537314,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":33.333333333333336,"upperBound":33.333333333333336,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":13.918439716312058,"upperBound":76.41843971631207,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":32.89473684210526,"upperBound":82.89473684210526,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":33.62068965517241,"upperBound":83.62068965517241,"provisional":true,"configurationsDiffer":true}}},{"modelId":"claude-sonnet-5.5","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":96.05263157894737,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:b171d979-5ec1-45de-b8b9-db1ee65e77ec","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max%2C%20Default%20Fallback","retrievedAt":"2026-10-19","value":57.49999999999999,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.05263157894737,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5.5 (Max, Default Fallback) (Release claude-sonnet-5-5, Zeile claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/b171d979-5ec1-45de-b8b9-db1ee65e77ec/claude-sonnet-5-5/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Modellstand: Claude Sonnet 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback. Angezeigter Wert: 57,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Claude Sonnet 5.5 (Max, Default Fallback) (release claude-sonnet-5-5, row claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/b171d979-5ec1-45de-b8b9-db1ee65e77ec/claude-sonnet-5-5/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Model snapshot: Claude Sonnet 5.5 (Max, Default Fallback). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback. Score display: 57.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5.5 (Max, Default Fallback) (Release claude-sonnet-5-5, Zeile claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Sonnet 5.5 (Max, Default Fallback) (release claude-sonnet-5-5, row claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Claude Sonnet 5.5 (Max, Default Fallback) (Release claude-sonnet-5-5, Zeile claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max, Default Fallback","en":"Exact AA configuration: Claude Sonnet 5.5 (Max, Default Fallback) (release claude-sonnet-5-5, row claude-sonnet-5-5, UUID b171d979-5ec1-45de-b8b9-db1ee65e77ec, reasoning=true, source reasoning-token configuration budget=0, suffix=Max, Default Fallback). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max, Default Fallback"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max, Default Fallback","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":94.44444444444444,"measurements":[{"observationId":"lmarena:webdev:overall:claude-sonnet-5.5-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1773.6614538733893,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":94.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 2.858 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5.5-xhigh. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,858 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5.5-xhigh. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.858 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh","en":"Arena rating from 2,858 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.858 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh","en":"Arena rating from 2,858 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":52.77777777777778,"measurements":[{"observationId":"lmarena:overall:claude-sonnet-5.5-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1475.2663811227833,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":52.77777777777778,"participants":37,"methodology":{"de":"Arena-Bewertung aus 6.254 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5.5-xhigh. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 6,254 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5.5-xhigh. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 6.254 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 6,254 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 6.254 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 6,254 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":43.333333333333336,"measurements":[{"observationId":"lmarena:vision_style_control:overall:claude-sonnet-5.5-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1266.188909365937,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":43.333333333333336,"participants":31,"methodology":{"de":"Arena-Bewertung aus 2.068 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/claude-sonnet-5.5-xhigh. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Modellstand: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,068 published pairwise votes.. Source model ID: lmarena/claude-sonnet-5.5-xhigh. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Model snapshot: claude-sonnet-5.5-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.068 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 2,068 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.068 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 2,068 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":93.73793436293437,"measurements":[{"observationId":"vals-terminal-bench-2-1:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":83.146,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.17857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 465.828. Standardfehler: 1.716. Kosten je Aufgabe in USD: 0.625494. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Compute: high. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 465.828. Standard error: 1.716. Cost per task in USD: 0.625494. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: high. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: high. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":64.141,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.29729729729729,"participants":38,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4928.927. Standardfehler: 1.01. Kosten je Aufgabe in USD: 16.513006. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4928.927. Standard error: 1.01. Cost per task in USD: 16.513006. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":91.93548387096774,"measurements":[{"observationId":"vals-terminal-bench-science:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":47.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.93548387096774,"participants":32,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9980.148. Standardfehler: 6.009. Kosten je Aufgabe in USD: 32.516187. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 9980.148. Standard error: 6.009. Cost per task in USD: 32.516187. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"vals-biomysterybench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":81.111,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":20,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1086.061. Standardfehler: 0.642. Kosten je Aufgabe in USD: 3.736494. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1086.061. Standard error: 0.642. Cost per task in USD: 3.736494. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-code-migration:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":69.825,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":63,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12004.483. Standardfehler: 4.262. Kosten je Aufgabe in USD: 75.826721. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 12004.483. Standard error: 4.262. Cost per task in USD: 75.826721. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":30.303030303030305,"measurements":[{"observationId":"vals-cyberbench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":59.584,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.303030303030305,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2032.625. Standardfehler: 5.212. Kosten je Aufgabe in USD: 4.417372. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2032.625. Standard error: 5.212. Cost per task in USD: 4.417372. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":96.61016949152543,"measurements":[{"observationId":"vals-emb:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":75.713,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.61016949152543,"participants":60,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2204.991. Standardfehler: 2.437. Kosten je Aufgabe in USD: 8.792863. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2204.991. Standard error: 2.437. Cost per task in USD: 8.792863. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":88.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":58.103,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1903.336. Standardfehler: 0.673. Kosten je Aufgabe in USD: 6.552431. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1903.336. Standard error: 0.673. Cost per task in USD: 6.552431. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":78.78787878787878,"measurements":[{"observationId":"vals-ioi:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":83.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.78787878787878,"participants":34,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2751.084. Standardfehler: 3.737. Kosten je Aufgabe in USD: 7.524252. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 2751.084. Standard error: 3.737. Cost per task in USD: 7.524252. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":50,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.917,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":62,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3464.426. Standardfehler: 1.362. Kosten je Aufgabe in USD: 16.612583. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3464.426. Standard error: 1.362. Cost per task in USD: 16.612583. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":48.077,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5588.989. Standardfehler: 3.473. Kosten je Aufgabe in USD: 14.826571. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5588.989. Standard error: 3.473. Cost per task in USD: 14.826571. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":85.07462686567165,"measurements":[{"observationId":"vals-medcode:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":52.92,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.07462686567165,"participants":68,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 239.171. Standardfehler: 2.119. Kosten je Aufgabe in USD: 0.391592. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 239.171. Standard error: 2.119. Cost per task in USD: 0.391592. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":97.05882352941177,"measurements":[{"observationId":"vals-medscribe:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":91.101,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.05882352941177,"participants":69,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 306.811. Standardfehler: 1.96. Kosten je Aufgabe in USD: 0.508604. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 306.811. Standard error: 1.96. Cost per task in USD: 0.508604. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":90,"measurements":[{"observationId":"vals-mysterymechanism:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":49.099,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90,"participants":21,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1410.236. Standardfehler: 3.363. Kosten je Aufgabe in USD: 3.45323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1410.236. Standard error: 3.363. Cost per task in USD: 3.45323. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":95.91836734693878,"measurements":[{"observationId":"vals-programbench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":6.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.91836734693878,"participants":50,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7602.078. Standardfehler: 1.748. Kosten je Aufgabe in USD: 35.860741. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7602.078. Standard error: 1.748. Cost per task in USD: 35.860741. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":97.43589743589743,"measurements":[{"observationId":"vals-proof-bench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":100,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.43589743589743,"participants":40,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 347.583. Standardfehler: 0. Kosten je Aufgabe in USD: 0.558327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 347.583. Standard error: 0. Cost per task in USD: 0.558327. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":86.20689655172414,"measurements":[{"observationId":"vals-sage:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":51.772,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.20689655172414,"participants":59,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 113.423. Standardfehler: 3.412. Kosten je Aufgabe in USD: 0.178566. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 113.423. Standard error: 3.412. Cost per task in USD: 0.178566. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":76.3157894736842,"measurements":[{"observationId":"vals-snap:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":67.185,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.3157894736842,"participants":39,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4527.815. Standardfehler: 1.221. Kosten je Aufgabe in USD: 4.245484. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 4527.815. Standard error: 1.221. Cost per task in USD: 4.245484. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":79.16666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":30.153,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.16666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6881.468. Standardfehler: 2.841. Kosten je Aufgabe in USD: 26.52067. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 6881.468. Standard error: 2.841. Cost per task in USD: 26.52067. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"vals-swebench-multimodal:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":58.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":14,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1452.55. Standardfehler: 2.253. Kosten je Aufgabe in USD: 4.528625. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 1452.55. Standard error: 2.253. Cost per task in USD: 4.528625. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":92.72727272727273,"measurements":[{"observationId":"vals-tax-agent-bench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":41.959,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.72727272727273,"participants":56,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3560.542. Standardfehler: 3.561. Kosten je Aufgabe in USD: 9.245466. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3560.542. Standard error: 3.561. Cost per task in USD: 9.245466. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-vibe-code-bench:overall:anthropic/claude-sonnet-5-5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":92.392,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":73,"methodology":{"de":"Kennung des Quellmodells: anthropic/claude-sonnet-5-5. Name des Quellmodells: Claude Sonnet 5.5. Anbieter laut Quelle: Anthropic. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3672.106. Standardfehler: 1.26. Kosten je Aufgabe in USD: 31.250578. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: anthropic/claude-sonnet-5-5. Source model name: Claude Sonnet 5.5. Provider reported by source: Anthropic. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 3672.106. Standard error: 1.26. Cost per task in USD: 31.250578. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Rechenaufwand: max. Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Compute: max. Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":25,"totalFamilies":71,"sourceCount":3,"coverage":0.38458049886621315,"lowerBound":27.920610794437238,"upperBound":89.46256090781591,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":2,"coverage":0.4555555555555557,"lowerBound":42.065319683407246,"upperBound":96.50976412785167,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":19.48717948717949,"upperBound":99.48717948717949,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":8.00438596491228,"upperBound":99.67105263157893,"provisional":true,"configurationsDiffer":true},"general":{"score":80.48611111111113,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":64.3888888888889,"upperBound":84.3888888888889,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":39.667253650304495,"upperBound":96.81011079316164,"provisional":true,"configurationsDiffer":true},"legal":{"score":68.33333333333334,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":34.16666666666667,"upperBound":84.16666666666667,"provisional":true,"configurationsDiffer":true},"health":{"score":91.06672519754171,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":60.711150131694474,"upperBound":94.04448346502781,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":30.303030303030305,"upperBound":30.303030303030305,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":10.833333333333334,"upperBound":85.83333333333333,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":38.1578947368421,"upperBound":88.15789473684211,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":43.10344827586207,"upperBound":93.10344827586206,"provisional":true,"configurationsDiffer":true}}},{"modelId":"cohere-c4ai-aya-expanse-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-c4ai-aya-expanse-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-c4ai-aya-vision-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-c4ai-aya-vision-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-command-a-plus-05-2026","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-command-a-reasoning-08-2025","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-command-a-translate-08-2025","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-command-a-vision-07-2025","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-command-r7b-arabic-02-2025","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"cohere-north-mini-code-1-0","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"command-a","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":20.28985507246377,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:3bc32f13-5afa-4e28-bce1-10e57376686b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":36.4625850340136,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.28985507246377,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Command A (Release command-a, Zeile command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/3bc32f13-5afa-4e28-bce1-10e57376686b/command-a/command-a. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Modellstand: Command A. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 36,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Command A (release command-a, row command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/3bc32f13-5afa-4e28-bce1-10e57376686b/command-a/command-a. Source model name: Command A. Provider reported by source: Cohere. Model snapshot: Command A. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 36.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Command A (Release command-a, Zeile command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Command A (release command-a, row command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Command A (Release command-a, Zeile command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Command A (release command-a, row command-a, UUID 3bc32f13-5afa-4e28-bce1-10e57376686b, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":7.894736842105263,"measurements":[{"observationId":"vals-gpqa:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":48.485,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.894736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 11.49. Standardfehler: 2.276. Kosten je Aufgabe in USD: 0.010094. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 11.49. Standard error: 2.276. Cost per task in USD: 0.010094. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":0,"measurements":[{"observationId":"vals-swebench:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":7.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":60,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 227.531. Standardfehler: 1.201. Kosten je Aufgabe in USD: 0.21024. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 227.531. Standard error: 1.201. Cost per task in USD: 0.21024. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":0,"measurements":[{"observationId":"vals-terminal-bench-2:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":2.247,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":43,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 760.674. Standardfehler: 1.58. Kosten je Aufgabe in USD: 2.436871. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 760.674. Standard error: 1.58. Cost per task in USD: 2.436871. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":5.405405405405405,"measurements":[{"observationId":"vals-corp-fin-v2:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":45.96,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.405405405405405,"participants":75,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 14.146. Standardfehler: 0.972. Kosten je Aufgabe in USD: 0.206138. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 14.146. Standard error: 0.972. Cost per task in USD: 0.206138. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":3.896103896103896,"measurements":[{"observationId":"vals-lcb:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":35.071,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.896103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 11.746. Standardfehler: 1.086. Kosten je Aufgabe in USD: 0.006568. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 11.746. Standard error: 1.086. Cost per task in USD: 0.006568. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"vals-legal-bench:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.697,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.76923076923077,"participants":79,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 0.797. Standardfehler: 0.422. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 0.797. Standard error: 0.422. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":13.636363636363637,"measurements":[{"observationId":"vals-math500:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":76.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.636363636363637,"participants":23,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 8.665. Standardfehler: 1.904. Kosten je Aufgabe in USD: 0.005983. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 8.665. Standard error: 1.904. Cost per task in USD: 0.005983. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":17.94871794871795,"measurements":[{"observationId":"vals-medqa:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":80.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.94871794871795,"participants":40,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 2.908. Standardfehler: 0.361. Kosten je Aufgabe in USD: 0.003459. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 2.908. Standard error: 0.361. Cost per task in USD: 0.003459. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":10.344827586206897,"measurements":[{"observationId":"vals-mgsm:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":85.709,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.344827586206897,"participants":30,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 8.36. Standardfehler: 2.016. Kosten je Aufgabe in USD: 0.003732. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 8.36. Standard error: 2.016. Cost per task in USD: 0.003732. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":5.194805194805195,"measurements":[{"observationId":"vals-mmlu-pro:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":69.172,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.194805194805195,"participants":78,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 9.853. Standardfehler: 0.463. Kosten je Aufgabe in USD: 0.007902. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 9.853. Standard error: 0.463. Cost per task in USD: 0.007902. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":5.2631578947368425,"measurements":[{"observationId":"vals-tax-eval-v2:overall:cohere/command-a-03-2025","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":61.366,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.2631578947368425,"participants":77,"methodology":{"de":"Kennung des Quellmodells: cohere/command-a-03-2025. Name des Quellmodells: Command A. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 8000. Latenz in Sekunden: 11.214. Standardfehler: 0.952. Kosten je Aufgabe in USD: 0.006484. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-a-03-2025. Source model name: Command A. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 8000. Latency in seconds: 11.214. Standard error: 0.952. Cost per task in USD: 0.006484. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 8000","en":"Temperature: 0.3. Maximum output tokens: 8000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":2,"coverage":0.14178004535147395,"lowerBound":1.900625369389108,"upperBound":87.72262083424171,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":1,"coverage":0.11111111111111113,"lowerBound":0.2597402597402598,"upperBound":89.14862914862914,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0.5639097744360902,"upperBound":93.42105263157895,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":4.7962382445141065,"upperBound":64.7962382445141,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":1.7316017316017316,"upperBound":68.3982683982684,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.057971014492754,"upperBound":84.05797101449275,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":1.5240804714488925,"upperBound":72.95265190002033,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":7.692307692307692,"upperBound":82.6923076923077,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":5.982905982905983,"upperBound":72.64957264957266,"provisional":true,"configurationsDiffer":true}}},{"modelId":"command-r","contributions":[{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"vals-legal-bench:overall:cohere/command-r","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":32.965,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":79,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r. Name des Quellmodells: Command R. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 0.244. Standardfehler: 0.71. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r. Source model name: Command R. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 0.244. Standard error: 0.71. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.01785714285714286,"lowerBound":0,"upperBound":98.21428571428571,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":0,"upperBound":75,"provisional":true,"configurationsDiffer":true}}},{"modelId":"command-r-plus","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":0,"measurements":[{"observationId":"vals-gpqa:overall:cohere/command-r-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":31.06,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":77,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r-plus. Name des Quellmodells: Command R+. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 8.194. Standardfehler: 2.28. Kosten je Aufgabe in USD: 0.007438. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r-plus. Source model name: Command R+. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 8.194. Standard error: 2.28. Cost per task in USD: 0.007438. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"vals-lcb:overall:cohere/command-r-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":18.238,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":78,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r-plus. Name des Quellmodells: Command R+. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 5.463. Standardfehler: 1.016. Kosten je Aufgabe in USD: 0.003267. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r-plus. Source model name: Command R+. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 5.463. Standard error: 1.016. Cost per task in USD: 0.003267. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"vals-legal-bench:overall:cohere/command-r-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":68.287,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":79,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r-plus. Name des Quellmodells: Command R+. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 0.396. Standardfehler: 0.65. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r-plus. Source model name: Command R+. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 0.396. Standard error: 0.65. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"vals-medqa:overall:cohere/command-r-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":2.651,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":40,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r-plus. Name des Quellmodells: Command R+. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 6.056. Standardfehler: 0.29. Kosten je Aufgabe in USD: 0.003456. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r-plus. Source model name: Command R+. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 6.056. Standard error: 0.29. Cost per task in USD: 0.003456. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"vals-mmlu-pro:overall:cohere/command-r-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":44.003,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":78,"methodology":{"de":"Kennung des Quellmodells: cohere/command-r-plus. Name des Quellmodells: Command R+. Anbieter laut Quelle: Cohere. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.3. Maximale Ausgabetoken: 4000. Latenz in Sekunden: 4.419. Standardfehler: 0.479. Kosten je Aufgabe in USD: 0.00566. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: cohere/command-r-plus. Source model name: Command R+. Provider reported by source: Cohere. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.3. Maximum output tokens: 4000. Latency in seconds: 4.419. Standard error: 0.479. Cost per task in USD: 0.00566. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 4000","en":"Temperature: 0.3. Maximum output tokens: 4000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":5,"totalFamilies":71,"sourceCount":1,"coverage":0.07534013605442177,"lowerBound":0.13736263736263737,"upperBound":92.60334903192046,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0,"upperBound":93.33333333333333,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0,"upperBound":92.85714285714286,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0,"upperBound":66.66666666666667,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":1.923076923076923,"upperBound":76.92307692307692,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0,"upperBound":66.66666666666667,"provisional":true,"configurationsDiffer":true}}},{"modelId":"command-r7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepreinforce-ornith-1.0-31b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepreinforce-ornith-1.0-35b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepreinforce-ornith-1.0-397b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepreinforce-ornith-1.0-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepreinforce-ornith-1.5-35b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-ocr-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-r1-distill-qwen-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-v4-flash-0731","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-v4-flash-vision-exp","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-v4-pro-0423","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-deepseek-v4-pro-0813","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-r1","contributions":[{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":20,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-deepseek-r1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":25.33,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":20,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Kosten je Aufgabe in USD: 0.9364333333333333. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-664f99d510f5. Prüfsumme der Quelldaten: 664f99d510f5321c20560c70c760680b5d40263821b082cc494f3bdfae6cc0cb. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Cost per task in USD: 0.9364333333333333. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-664f99d510f5. Source payload hash: 664f99d510f5321c20560c70c760680b5d40263821b082cc494f3bdfae6cc0cb. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":20,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-deepseek-r1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":23.53,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":20,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Kosten je Aufgabe in USD: 0.1788235294117647. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-6fc49efd47b7. Prüfsumme der Quelldaten: 6fc49efd47b70abe914ca04758101a25125a9e8a20c5a6c90019906650d6beee. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Cost per task in USD: 0.1788235294117647. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-6fc49efd47b7. Source payload hash: 6fc49efd47b70abe914ca04758101a25125a9e8a20c5a6c90019906650d6beee. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-deepseek-r1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":38.11,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Kosten je Aufgabe in USD: 0.2607166123778502. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-b17824331ef0. Prüfsumme der Quelldaten: b17824331ef027e156721f99110a57b278ee78be01b8959a892bacfb3502ff67. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Cost per task in USD: 0.2607166123778502. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-b17824331ef0. Source payload hash: b17824331ef027e156721f99110a57b278ee78be01b8959a892bacfb3502ff67. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"curated:scale-multinrc-deepseek-r1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":27.58,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-r1-0528. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 27,58 % ± 2,7. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-ee6d5a59e0c6. Prüfsumme der Quelldaten: ee6d5a59e0c65ff18a7d24b1e80e2265bd6fdefe8e24bd84ffb982ddcf754154. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: DeepSeek. Model snapshot: deepseek-r1-0528. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 27.58% ± 2.7. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-ee6d5a59e0c6. Source payload hash: ee6d5a59e0c65ff18a7d24b1e80e2265bd6fdefe8e24bd84ffb982ddcf754154. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":16.216216216216218,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":54.118,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.216216216216218,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 46.774. Standardfehler: 0.979. Kosten je Aufgabe in USD: 0.086732. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 46.774. Standard error: 0.979. Cost per task in USD: 0.086732. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":23.376623376623378,"measurements":[{"observationId":"vals-lcb:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":70.221,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.376623376623378,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 86.074. Standardfehler: 1.15. Kosten je Aufgabe in USD: 0.040788. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 86.074. Standard error: 1.15. Cost per task in USD: 0.040788. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":5.128205128205129,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":67.323,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.128205128205129,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 2048. Latenz in Sekunden: 18.549. Standardfehler: 0.535. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 2048. Latency in seconds: 18.549. Standard error: 0.535. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 2048","en":"Temperature: 1. Maximum output tokens: 2048"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 2048","en":"Temperature: 1. Maximum output tokens: 2048"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":59.09090909090909,"measurements":[{"observationId":"vals-math500:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":92.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.09090909090909,"participants":23,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 156.474. Standardfehler: 2.364. Kosten je Aufgabe in USD: 0.010886. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 32768. Latency in seconds: 156.474. Standard error: 2.364. Cost per task in USD: 0.010886. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Temperature: 0. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Temperature: 0. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":43.58974358974359,"measurements":[{"observationId":"vals-medqa:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":90.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.58974358974359,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 41.569. Standardfehler: 0.518. Kosten je Aufgabe in USD: 0.005903. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 4096. Latency in seconds: 41.569. Standard error: 0.518. Cost per task in USD: 0.005903. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 4096","en":"Temperature: 1. Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 4096","en":"Temperature: 1. Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":82.75862068965517,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":92.254,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.75862068965517,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 10.167. Standardfehler: 1.655. Kosten je Aufgabe in USD: 0.004123. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 10.167. Standard error: 1.655. Cost per task in USD: 0.004123. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":38.96103896103896,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":83.184,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.96103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 27.278. Standardfehler: 0.444. Kosten je Aufgabe in USD: 0.012339. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 27.278. Standard error: 0.444. Cost per task in USD: 0.012339. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":44.73684210526316,"measurements":[{"observationId":"vals-tax-eval-v2:overall:fireworks/deepseek-r1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.281,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-r1. Name des Quellmodells: DeepSeek-R1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 152.958. Standardfehler: 0.874. Kosten je Aufgabe in USD: 0.012239. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-r1. Source model name: DeepSeek-R1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 152.958. Standard error: 0.874. Cost per task in USD: 0.012239. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":3,"coverage":0.1448979591836735,"lowerBound":5.2165934765105115,"upperBound":90.72679755814316,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":2,"coverage":0.20000000000000007,"lowerBound":2.891774891774893,"upperBound":82.89177489177489,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":2.597402597402598,"upperBound":88.31168831168831,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":28.369905956112852,"upperBound":88.36990595611286,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":12.987012987012985,"upperBound":79.65367965367966,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":1.6666666666666667,"upperBound":93.33333333333333,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":8.70757976021134,"upperBound":80.13615118878278,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":1.2820512820512822,"upperBound":76.28205128205128,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":14.529914529914532,"upperBound":81.1965811965812,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-r1-0528","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":27.536231884057973,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:a83f84b3-473a-4276-9ae1-8909da723159","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":39.5918367346939,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":27.536231884057973,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek R1 0528 (May '25) (Release deepseek-r1, Zeile deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/a83f84b3-473a-4276-9ae1-8909da723159/deepseek-r1/deepseek-r1. Name des Quellmodells: DeepSeek-R1 0528. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek R1 0528 (May '25). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 39,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek R1 0528 (May '25) (release deepseek-r1, row deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/a83f84b3-473a-4276-9ae1-8909da723159/deepseek-r1/deepseek-r1. Source model name: DeepSeek-R1 0528. Provider reported by source: DeepSeek. Model snapshot: DeepSeek R1 0528 (May '25). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 39.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek R1 0528 (May '25) (Release deepseek-r1, Zeile deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: DeepSeek R1 0528 (May '25) (release deepseek-r1, row deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek R1 0528 (May '25) (Release deepseek-r1, Zeile deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: DeepSeek R1 0528 (May '25) (release deepseek-r1, row deepseek-r1, UUID a83f84b3-473a-4276-9ae1-8909da723159, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.3933747412008282,"upperBound":98.96480331262941,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":5.507246376811595,"upperBound":85.5072463768116,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-r1-distill-llama-70b","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":4.3478260869565215,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:4a845d7b-a52d-43bb-80b7-b58c7a0c155e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":27.5510204081633,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.3478260869565215,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek R1 Distill Llama 70B (Release deepseek-r1-distill-llama-70b, Zeile deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/4a845d7b-a52d-43bb-80b7-b58c7a0c155e/deepseek-r1-distill-llama-70b/deepseek-r1-distill-llama-70b. Name des Quellmodells: DeepSeek-R1-Distill-Llama-70B. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek R1 Distill Llama 70B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 27,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek R1 Distill Llama 70B (release deepseek-r1-distill-llama-70b, row deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/4a845d7b-a52d-43bb-80b7-b58c7a0c155e/deepseek-r1-distill-llama-70b/deepseek-r1-distill-llama-70b. Source model name: DeepSeek-R1-Distill-Llama-70B. Provider reported by source: DeepSeek. Model snapshot: DeepSeek R1 Distill Llama 70B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 27.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek R1 Distill Llama 70B (Release deepseek-r1-distill-llama-70b, Zeile deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: DeepSeek R1 Distill Llama 70B (release deepseek-r1-distill-llama-70b, row deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek R1 Distill Llama 70B (Release deepseek-r1-distill-llama-70b, Zeile deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: DeepSeek R1 Distill Llama 70B (release deepseek-r1-distill-llama-70b, row deepseek-r1-distill-llama-70b, UUID 4a845d7b-a52d-43bb-80b7-b58c7a0c155e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.062111801242236024,"upperBound":98.63354037267082,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.8695652173913043,"upperBound":80.8695652173913,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v3","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":77.7,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":29.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 29,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 29.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.5829479653698352,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":3.658536585365854,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-28.39,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.658536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":15.942028985507246,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":34.7619047619048,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.942028985507246,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 34,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 34.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-41.65,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":10.526315789473685,"measurements":[{"observationId":"vals-gpqa:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":54.546,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 21.282. Standardfehler: 2.504. Kosten je Aufgabe in USD: 0.001468. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 21.282. Standard error: 2.504. Cost per task in USD: 0.001468. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-assistantbench","area":"agentic","label":{"de":"AssistantBench","en":"AssistantBench"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-assistantbench-browser-use-overall-deepseek-v3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/assistantbench","benchmarkId":"hal-assistantbench","benchmarkVersion":"HAL 33-task public set","comparisonCohort":"hal-assistantbench-browser-use-overall","retrievedAt":"2026-08-05","value":2.03,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":5,"methodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Kennung des Quellmodells: curated/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-v3-0324. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Kosten je Aufgabe in USD: 0.38363636363636366. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-9a5202122ebd. Prüfsumme der Quelldaten: 9a5202122ebda67b0616c9e02bb7338b3a7efbd02c0281f38afbc539486363dc. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Source model ID: curated/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: deepseek-v3-0324. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Cost per task in USD: 0.38363636363636366. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-9a5202122ebd. Source payload hash: 9a5202122ebda67b0616c9e02bb7338b3a7efbd02c0281f38afbc539486363dc. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33."},"scoreRunConfiguration":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Harness: Browser-Use. Agent: Browser-Use","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Harness: Browser-Use. Agent: Browser-Use"},"scoreComparisonSetup":{"publicationCohort":"hal-assistantbench-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":80,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-deepseek-v3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":32.33,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":80,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Kosten je Aufgabe in USD: 0.7158. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-4f73f352634c. Prüfsumme der Quelldaten: 4f73f352634c61b68118cfe631a33093434e04a6de3ca7be5e55ee7a038f4074. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Cost per task in USD: 0.7158. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-4f73f352634c. Source payload hash: 4f73f352634c61b68118cfe631a33093434e04a6de3ca7be5e55ee7a038f4074. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-deepseek-v3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":15.69,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Kosten je Aufgabe in USD: 0.02049019607843137. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-205da614cb5f. Prüfsumme der Quelldaten: 205da614cb5f85706a2bf9b0e8c27e8a75cf8c503ee510834f6ee27bdc15e656. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Cost per task in USD: 0.02049019607843137. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-205da614cb5f. Source payload hash: 205da614cb5f85706a2bf9b0e8c27e8a75cf8c503ee510834f6ee27bdc15e656. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":20,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-deepseek-v3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":39.09,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":20,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Kosten je Aufgabe in USD: 0.039348534201954395. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-d7390cf317e3. Prüfsumme der Quelldaten: d7390cf317e3a69cc8a9318b4278c5077dfef4f126088a7f33bd3bde371e78f9. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Cost per task in USD: 0.039348534201954395. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-d7390cf317e3. Source payload hash: d7390cf317e3a69cc8a9318b4278c5077dfef4f126088a7f33bd3bde371e78f9. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":2.9000000000000004,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 2,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 2.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":0,"measurements":[{"observationId":"curated:hal-swebench-verified-mini-swe-agent-overall-deepseek-v3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/swebench_verified_mini","benchmarkId":"hal-swebench-verified-mini","benchmarkVersion":"50-task random verified subset","comparisonCohort":"hal-swebench-verified-mini-swe-agent-overall","retrievedAt":"2026-08-05","value":24,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":5,"methodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Kennung des Quellmodells: curated/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SWE-Agent. Agent: SWE-Agent. Kosten je Aufgabe in USD: 0.2354. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-69f60128a276. Prüfsumme der Quelldaten: 69f60128a276569b958f9a6412d4d7437b18e233dd872ac2800cfd8194e19711. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Source model ID: curated/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Evaluator: Princeton HAL. Dataset type: public. Harness: SWE-Agent. Agent: SWE-Agent. Cost per task in USD: 0.2354. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-69f60128a276. Source payload hash: 69f60128a276569b958f9a6412d4d7437b18e233dd872ac2800cfd8194e19711. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50."},"scoreRunConfiguration":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Harness: SWE-Agent. Agent: SWE-Agent","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Harness: SWE-Agent. Agent: SWE-Agent"},"scoreComparisonSetup":{"publicationCohort":"hal-swebench-verified-mini-swe-agent-overall","datasetType":"public","harness":"SWE-Agent","agent":"SWE-Agent","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:43fc5506-c5ed-4dee-9b85-962bf7ae3986","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43fc5506-c5ed-4dee-9b85-962bf7ae3986/deepseek-v3/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 (Dec '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 (Dec '24) (Release deepseek-v3, Zeile deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 (Dec '24) (release deepseek-v3, row deepseek-v3, UUID 43fc5506-c5ed-4dee-9b85-962bf7ae3986, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":13.513513513513514,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":52.486,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.513513513513514,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 28.322. Standardfehler: 0.98. Kosten je Aufgabe in USD: 0.052026. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 28.322. Standard error: 0.98. Cost per task in USD: 0.052026. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":37.17948717948718,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":80.762,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.17948717948718,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4.395. Standardfehler: 0.429. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4.395. Standard error: 0.429. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-math500:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":80.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":23,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 7.94. Standardfehler: 1.775. Kosten je Aufgabe in USD: 0.000563. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 7.94. Standard error: 1.775. Cost per task in USD: 0.000563. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":20.512820512820515,"measurements":[{"observationId":"vals-medqa:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":80.9,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.512820512820515,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 6.816. Standardfehler: 0.359. Kosten je Aufgabe in USD: 0.000578. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 6.816. Standard error: 0.359. Cost per task in USD: 0.000578. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":79.3103448275862,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":92.146,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.3103448275862,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 15.681. Standardfehler: 1.663. Kosten je Aufgabe in USD: 0.000364. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 15.681. Standard error: 1.663. Cost per task in USD: 0.000364. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":10.38961038961039,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":73.82,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.38961038961039,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 11.36. Standardfehler: 0.424. Kosten je Aufgabe in USD: 0.001338. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 11.36. Standard error: 0.424. Cost per task in USD: 0.001338. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":15.789473684210526,"measurements":[{"observationId":"vals-tax-eval-v2:overall:fireworks/deepseek-v3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":67.907,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.789473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3. Name des Quellmodells: DeepSeek-V3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 27.696. Standardfehler: 0.918. Kosten je Aufgabe in USD: 0.00056. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3. Source model name: DeepSeek-V3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 27.696. Standard error: 0.918. Cost per task in USD: 0.00056. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":22,"totalFamilies":71,"sourceCount":3,"coverage":0.22721088435374157,"lowerBound":4.841159825595927,"upperBound":82.12007139022177,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":4,"totalFamilies":15,"sourceCount":2,"coverage":0.20000000000000007,"lowerBound":1.922764227642277,"upperBound":81.92276422764228,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":2.66825600586833,"upperBound":59.81111314872547,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":21.3166144200627,"upperBound":81.3166144200627,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":14.341146658219829,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":9.56076443881322,"upperBound":42.89409777214655,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":8.800813008130081,"upperBound":67.1341463414634,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":3.1884057971014492,"upperBound":83.18840579710145,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":4.186141028246292,"upperBound":75.61471245681773,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":9.294871794871796,"upperBound":84.2948717948718,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":6.837606837606838,"upperBound":73.50427350427351,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v3-0324","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":68.11,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":40.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 40,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 40.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":4.878048780487805,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.2729859795469024,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.878048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":14.024390243902438,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.024390243902438,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":69.71,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":31.884057971014492,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":41.0204081632653,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.884057971014492,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 41,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 41.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-40.666666666666664,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":11.842105263157896,"measurements":[{"observationId":"vals-gpqa:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":61.616,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.842105263157896,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 23.475. Standardfehler: 2.447. Kosten je Aufgabe in USD: 0.001552. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 23.475. Standard error: 2.447. Cost per task in USD: 0.001552. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.74,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:75e1c197-f239-4361-a9d6-66dccfead236","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3 0324. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/75e1c197-f239-4361-a9d6-66dccfead236/deepseek-v3-0324/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3 0324. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3 0324 (Release deepseek-v3-0324, Zeile deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: DeepSeek V3 0324 (release deepseek-v3-0324, row deepseek-v3-0324, UUID 75e1c197-f239-4361-a9d6-66dccfead236, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":20.27027027027027,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":54.74,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.27027027027027,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 36.732. Standardfehler: 0.978. Kosten je Aufgabe in USD: 0.052049. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 36.732. Standard error: 0.978. Cost per task in USD: 0.052049. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":16.883116883116884,"measurements":[{"observationId":"vals-lcb:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":65.478,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.883116883116884,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 26.93. Standardfehler: 1.164. Kosten je Aufgabe in USD: 0.001776. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 26.93. Standard error: 1.164. Cost per task in USD: 0.001776. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":19.23076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":77.727,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.23076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 7.712. Standardfehler: 0.418. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 7.712. Standard error: 0.418. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":40.90909090909091,"measurements":[{"observationId":"vals-math500:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":88.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.90909090909091,"participants":23,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 12.545. Standardfehler: 1.421. Kosten je Aufgabe in USD: 0.00074. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 12.545. Standard error: 1.421. Cost per task in USD: 0.00074. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"vals-medqa:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":82,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 12.876. Standardfehler: 0.351. Kosten je Aufgabe in USD: 0.000614. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 12.876. Standard error: 0.351. Cost per task in USD: 0.000614. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":68.96551724137932,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":91.673,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.96551724137932,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 18.032. Standardfehler: 1.696. Kosten je Aufgabe in USD: 0.000395. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 18.032. Standard error: 1.696. Cost per task in USD: 0.000395. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":20.77922077922078,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":79.474,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.77922077922078,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 25.051. Standardfehler: 0.396. Kosten je Aufgabe in USD: 0.00147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 25.051. Standard error: 0.396. Cost per task in USD: 0.00147. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":29.605263157894736,"measurements":[{"observationId":"vals-tax-eval-v2:overall:fireworks/deepseek-v3-0324","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.096,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.605263157894736,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3-0324. Name des Quellmodells: DeepSeek-V3 0324. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 34.009. Standardfehler: 0.883. Kosten je Aufgabe in USD: 0.000652. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3-0324. Source model name: DeepSeek-V3 0324. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 8192. Latency in seconds: 34.009. Standard error: 0.883. Cost per task in USD: 0.000652. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":18,"totalFamilies":71,"sourceCount":2,"coverage":0.2081632653061225,"lowerBound":5.030616365040272,"upperBound":84.21428983442803,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":1.7149720198500695,"upperBound":91.71497201985005,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":5.375481386392812,"upperBound":62.51833852924995,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":21.974921630094045,"upperBound":81.97492163009404,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":20.755464048146973,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":13.836976032097983,"upperBound":47.17030936543132,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":1.524390243902439,"upperBound":76.52439024390245,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.3768115942028984,"upperBound":86.3768115942029,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":7.125076204023572,"upperBound":78.553647632595,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":4.8076923076923075,"upperBound":79.8076923076923,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":7.692307692307692,"upperBound":74.35897435897436,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v3.1","contributions":[{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":14.285714285714286,"measurements":[{"observationId":"vals-terminal-bench-2:overall:fireworks/deepseek-v3p1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":22.472,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":43,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p1. Name des Quellmodells: DeepSeek-V3.1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Maximale Ausgabetoken: 163840. Latenz in Sekunden: 837.847. Standardfehler: 4.449. Kosten je Aufgabe in USD: 0.501536. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p1. Source model name: DeepSeek-V3.1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Maximum output tokens: 163840. Latency in seconds: 837.847. Standard error: 4.449. Cost per task in USD: 0.501536. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 163840","en":"Temperature: 1. Maximum output tokens: 163840"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 163840","en":"Temperature: 1. Maximum output tokens: 163840"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":12.162162162162161,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/deepseek-v3p1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":51.476,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.162162162162161,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p1. Name des Quellmodells: DeepSeek-V3.1. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 62.999. Standardfehler: 0.982. Kosten je Aufgabe in USD: 0.037369. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p1. Source model name: DeepSeek-V3.1. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 62.999. Standard error: 0.982. Cost per task in USD: 0.037369. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":75,"measurements":[{"observationId":"vals-poker-agent:overall:fireworks/deepseek-v3p1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1058.2325209691314,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":9,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p1. Name des Quellmodells: DeepSeek-V3.1. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 3.9977358921550272. Standardfehler: 0. Kosten je Aufgabe in USD: 0.007085498012100262. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p1. Source model name: DeepSeek-V3.1. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 3.9977358921550272. Standard error: 0. Cost per task in USD: 0.007085498012100262. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":3,"totalFamilies":71,"sourceCount":1,"coverage":0.016950113378684812,"lowerBound":0.5818701354415641,"upperBound":98.88685879757308,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.011111111111111113,"lowerBound":0.15873015873015878,"upperBound":99.04761904761905,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":6.25,"upperBound":97.91666666666666,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":1.7374517374517373,"upperBound":87.45173745173746,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v3.1-terminus","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-v3.2","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":26.923076923076923,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:d621247c-d47e-458c-82cb-a166bc3b37e5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":14.5280235988201,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.923076923076923,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3.2 (Reasoning) (Release deepseek-v3-2, Zeile deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/d621247c-d47e-458c-82cb-a166bc3b37e5/deepseek-v3-2/deepseek-v3-2-reasoning. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V3.2 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 14,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V3.2 (Reasoning) (release deepseek-v3-2, row deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/d621247c-d47e-458c-82cb-a166bc3b37e5/deepseek-v3-2/deepseek-v3-2-reasoning. Source model name: DeepSeek-V3.2. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V3.2 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 14.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V3.2 (Reasoning) (Release deepseek-v3-2, Zeile deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: DeepSeek V3.2 (Reasoning) (release deepseek-v3-2, row deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V3.2 (Reasoning) (Release deepseek-v3-2, Zeile deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: DeepSeek V3.2 (Reasoning) (release deepseek-v3-2, row deepseek-v3-2-reasoning, UUID d621247c-d47e-458c-82cb-a166bc3b37e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":27.63157894736842,"measurements":[{"observationId":"vals-gpqa:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":76.263,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.63157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480. Latenz in Sekunden: 46.828. Standardfehler: 2.161. Kosten je Aufgabe in USD: 0.00065. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480. Latency in seconds: 46.828. Standard error: 2.161. Cost per task in USD: 0.00065. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"none","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":30.95238095238095,"measurements":[{"observationId":"vals-terminal-bench-2:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":34.831,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.952380952380953,"participants":43,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480. Latenz in Sekunden: 804.541. Standardfehler: 5.079. Kosten je Aufgabe in USD: 0.063298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480. Latency in seconds: 804.541. Standard error: 5.079. Cost per task in USD: 0.063298. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"none","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":9.45945945945946,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":47.941,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.45945945945946,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480. Latenz in Sekunden: 133.787. Standardfehler: 0.98. Kosten je Aufgabe in USD: 0.036826. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480. Latency in seconds: 133.787. Standard error: 0.98. Cost per task in USD: 0.036826. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"none","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":20.77922077922078,"measurements":[{"observationId":"vals-lcb:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":69.856,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.77922077922078,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 39.343. Standardfehler: 1.198. Kosten je Aufgabe in USD: 0.000677. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 39.343. Standard error: 1.198. Cost per task in USD: 0.000677. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":3.8461538461538463,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":65.962,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.8461538461538463,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Latenz in Sekunden: 2.185. Standardfehler: 0.419. Kosten je Aufgabe in USD: 0.000086. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Latency in seconds: 2.185. Standard error: 0.419. Cost per task in USD: 0.000086. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"vals-medqa:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":89.45,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.46153846153846,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480. Latenz in Sekunden: 13.297. Standardfehler: 0.28. Kosten je Aufgabe in USD: 0.000234. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480. Latency in seconds: 13.297. Standard error: 0.28. Cost per task in USD: 0.000234. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: none. Temperatur: 1. Maximale Ausgabetoken: 20480","en":"Reasoning effort: none. Temperature: 1. Maximum output tokens: 20480"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"none","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":58.62068965517241,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":90.873,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.62068965517241,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Latenz in Sekunden: 8.966. Standardfehler: 1.768. Kosten je Aufgabe in USD: 0.00015. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Latency in seconds: 8.966. Standard error: 1.768. Cost per task in USD: 0.00015. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":37.66233766233766,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/deepseek-v3p2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":83.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.66233766233766,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/deepseek-v3p2. Name des Quellmodells: DeepSeek-V3.2. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 40.772. Standardfehler: 0.381. Kosten je Aufgabe in USD: 0.000283. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/deepseek-v3p2. Source model name: DeepSeek-V3.2. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 40.772. Standard error: 0.381. Cost per task in USD: 0.000283. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":2,"coverage":0.10657596371882087,"lowerBound":3.2398659858473184,"upperBound":92.58226961396524,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":1,"coverage":0.0777777777777778,"lowerBound":1.7291967291967296,"upperBound":93.95141895141894,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.9736842105263155,"upperBound":94.83082706766918,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.724137931034482,"upperBound":91.72413793103448,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":12.554112554112555,"upperBound":79.22077922077922,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":2.2435897435897436,"upperBound":93.9102564102564,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":1.3513513513513513,"upperBound":87.06563706563708,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":0.9615384615384615,"upperBound":75.96153846153847,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":12.82051282051282,"upperBound":79.48717948717949,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v3.2-exp","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"deepseek-v4-flash","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1258.64,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":79.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 79,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 79.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":53.96771855133674,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 54,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 54.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":46.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":11,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 11,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 11.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1433.74,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":-14.283333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":38.5542168674699,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 38,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 38.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":13.88888888888889,"measurements":[{"observationId":"lmarena:webdev:overall:deepseek-v4-flash-high-preview","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1430.6320866639408,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":13.88888888888889,"participants":37,"methodology":{"de":"Arena-Bewertung aus 10.152 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/deepseek-v4-flash-high-preview. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-v4-flash-high-preview. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,152 published pairwise votes.. Source model ID: lmarena/deepseek-v4-flash-high-preview. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: deepseek-v4-flash-high-preview. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.152 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 10,152 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.152 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 10,152 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:fe4c0848-e284-4e52-a79d-cdc28392f1a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":12.1212121212121,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Name des Quellmodells: DeepSeek-V4-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 12,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fe4c0848-e284-4e52-a79d-cdc28392f1a9/deepseek-v4-flash/deepseek-v4-flash. Source model name: DeepSeek-V4-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Flash 0731 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 12.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Flash 0731 (Max) (Release deepseek-v4-flash, Zeile deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Flash 0731 (Max) (release deepseek-v4-flash, row deepseek-v4-flash, UUID fe4c0848-e284-4e52-a79d-cdc28392f1a9, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":2,"coverage":0.07431972789115647,"lowerBound":3.89038677801744,"upperBound":96.4584139889018,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":3.202348690153569,"upperBound":93.20234869015356,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":19.860627177700348,"upperBound":84.14634146341464,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":13.821138211382113,"upperBound":80.48780487804879,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":17.58130081300813,"upperBound":92.58130081300813,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v4-pro","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":31.818181818181817,"measurements":[{"observationId":"research:deepseek-v4-pro:70","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9666666666666667,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":31.818181818181817,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: HjTDENSpy9sancDzr7pccY. Startzeit: 2026-06-17T00:34:52.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9666666666666667 und stderr=0.02002208376727653. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: HjTDENSpy9sancDzr7pccY. Started at 2026-06-17T00:34:52.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9666666666666667 and stderr=0.02002208376727653. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1256.94,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":80.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":56.711655463440025,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 56,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 56.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":11.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 11,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 11.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1450.47,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":54.8780487804878,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":0.8333333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.8780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"research:deepseek-v4-pro:66","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.19,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: DSyqeLiA9LLZF4iaxsTVHL. Startzeit: 2026-10-13T13:38:19.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.19 und stderr=0.039427724440366255. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: DSyqeLiA9LLZF4iaxsTVHL. Started at 2026-10-13T13:38:19.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.19 and stderr=0.039427724440366255. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":27.77777777777778,"measurements":[{"observationId":"research:deepseek-v4-pro:68","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.46993987975951906,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":27.77777777777778,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: Sq9xqkVTmFvELpQy9KH5TM. Startzeit: 2026-08-27T19:30:29.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.46993987975951906 und stderr=0.015806515332355317. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: Sq9xqkVTmFvELpQy9KH5TM. Started at 2026-08-27T19:30:29.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.46993987975951906 and stderr=0.015806515332355317. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"research:deepseek-v4-pro:61","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.024390243902439025,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: RAZ6VMfGtQSE8EvUZRRz4Q. Startzeit: 2026-06-17T02:30:14.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.024390243902439025 und stderr=0.024390243902439025. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: RAZ6VMfGtQSE8EvUZRRz4Q. Started at 2026-06-17T02:30:14.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.024390243902439025 and stderr=0.024390243902439025. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:deepseek-v4-pro:60","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.45263157894736844,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: KUar3FaFjzq5SMKrEhtn8e. Startzeit: 2026-06-17T02:30:14.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.45263157894736844 und stderr=0.029536098269922092. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: KUar3FaFjzq5SMKrEhtn8e. Started at 2026-06-17T02:30:14.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.45263157894736844 and stderr=0.029536098269922092. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":46.2406015037594,"measurements":[{"observationId":"research:deepseek-v4-pro:63","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.8964646464646465,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":21.428571428571427,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: deepseek-v4-pro_max. Epoch-Modellbezeichnung: DeepSeek-V4-Pro. Anzeigenamenfeld: DeepSeek v4 Pro (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: LuZUnhD5rBLjRw36DaqfvZ. Startzeit: 2026-06-16T20:16:55.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.8964646464646465 und stderr=0.017473014662966007. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: deepseek-v4-pro_max; Epoch model label: DeepSeek-V4-Pro; Display label field: DeepSeek v4 Pro (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: LuZUnhD5rBLjRw36DaqfvZ. Started at 2026-06-16T20:16:55.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.8964646464646465 and stderr=0.017473014662966007. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":89.394,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 900.378. Standardfehler: 1.654. Kosten je Aufgabe in USD: 0.125847. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 256000. Latency in seconds: 900.378. Standard error: 1.654. Cost per task in USD: 0.125847. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":41.0101946246525,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":53.389830508474574,"measurements":[{"observationId":"vals-swebench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":77.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.389830508474574,"participants":60,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 634.611. Standardfehler: 1.872. Kosten je Aufgabe in USD: 0.436763. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 256000. Latency in seconds: 634.611. Standard error: 1.872. Cost per task in USD: 0.436763. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":57.42059568279082,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":14.1414141414141,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 14,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 14.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":56.18,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.80952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning: false. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 828.342. Standardfehler: 5.289. Kosten je Aufgabe in USD: 0.179637. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning: false. Maximum output tokens: 128000. Latency in seconds: 828.342. Standard error: 5.289. Cost per task in USD: 0.179637. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: false. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: false. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":50.187,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.642857142857142,"participants":57,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning: false. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1078.158. Standardfehler: 1.498. Kosten je Aufgabe in USD: 0.239191. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning: false. Maximum output tokens: 128000. Latency in seconds: 1078.158. Standard error: 1.498. Cost per task in USD: 0.239191. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning: false. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning: false. Maximale Ausgabetoken: 128000","en":"Reasoning: false. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":11.111,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.72972972972973,"participants":38,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 5146.973. Standardfehler: 1.336. Kosten je Aufgabe in USD: 3.452522. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 5146.973. Standard error: 1.336. Cost per task in USD: 3.452522. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":34.78260869565217,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:83173329-c09a-41f1-a028-a282a5f908d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":5.71428571428571,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.78260869565217,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 5,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/83173329-c09a-41f1-a028-a282a5f908d7/deepseek-v4-pro/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4 Pro 0813 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 5.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4 Pro 0813 (Max) (Release deepseek-v4-pro, Zeile deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4 Pro 0813 (Max) (release deepseek-v4-pro, row deepseek-v4-pro, UUID 83173329-c09a-41f1-a028-a282a5f908d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":45.16129032258065,"measurements":[{"observationId":"vals-code-migration:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":26.201,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.16129032258065,"participants":63,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1737.978. Standardfehler: 4.037. Kosten je Aufgabe in USD: 1.071016. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 1737.978. Standard error: 4.037. Cost per task in USD: 1.071016. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":52.7027027027027,"measurements":[{"observationId":"vals-corp-fin-v2:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.383,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.7027027027027,"participants":75,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 79.331. Standardfehler: 0.959. Kosten je Aufgabe in USD: 0.148855. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 79.331. Standard error: 0.959. Cost per task in USD: 0.148855. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":30.508474576271187,"measurements":[{"observationId":"vals-emb:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":51.623,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.508474576271187,"participants":60,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 822.144. Standardfehler: 3.053. Kosten je Aufgabe in USD: 0.796757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 822.144. Standard error: 3.053. Cost per task in USD: 0.796757. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":28.333333333333332,"measurements":[{"observationId":"vals-finance-agent-v2:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":44.083,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1526.183. Standardfehler: 0.653. Kosten je Aufgabe in USD: 0.879086. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 1526.183. Standard error: 0.653. Cost per task in USD: 0.879086. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"vals-lcb:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.484,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":78,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 715.504. Standardfehler: 0.953. Kosten je Aufgabe in USD: 0.109669. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 715.504. Standard error: 0.953. Cost per task in USD: 0.109669. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":55.73770491803279,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":3.75,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.73770491803279,"participants":62,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 401.522. Standardfehler: 1.431. Kosten je Aufgabe in USD: 0.737465. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 401.522. Standard error: 1.431. Cost per task in USD: 0.737465. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":34.61538461538461,"measurements":[{"observationId":"vals-legal-bench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":80.323,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.61538461538461,"participants":79,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 184.46. Standardfehler: 0.468. Kosten je Aufgabe in USD: 0.004788. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 256000. Latency in seconds: 184.46. Standard error: 0.468. Cost per task in USD: 0.004788. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":31.666666666666668,"measurements":[{"observationId":"vals-legal-research:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":23.077,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1024.619. Standardfehler: 2.928. Kosten je Aufgabe in USD: 0.682218. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 1024.619. Standard error: 2.928. Cost per task in USD: 0.682218. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":29.850746268656717,"measurements":[{"observationId":"vals-medcode:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.455,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.850746268656717,"participants":68,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 382.667. Standardfehler: 2.122. Kosten je Aufgabe in USD: 0.06071. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 382.667. Standard error: 2.122. Cost per task in USD: 0.06071. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":23.529411764705884,"measurements":[{"observationId":"vals-medscribe:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":75.144,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.529411764705884,"participants":69,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 345.723. Standardfehler: 2.002. Kosten je Aufgabe in USD: 0.053954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 345.723. Standard error: 2.002. Cost per task in USD: 0.053954. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"vals-mmlu-pro:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.249,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.42857142857143,"participants":78,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 577.617. Standardfehler: 0.335. Kosten je Aufgabe in USD: 0.027032. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 256000. Latency in seconds: 577.617. Standard error: 0.335. Cost per task in USD: 0.027032. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Reasoning effort: max. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 3778.098. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 3778.098. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":17.94871794871795,"measurements":[{"observationId":"vals-proof-bench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":16,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.94871794871795,"participants":40,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 423.45. Standardfehler: 3.685. Kosten je Aufgabe in USD: 0.033594. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 423.45. Standard error: 3.685. Cost per task in USD: 0.033594. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":44.827586206896555,"measurements":[{"observationId":"vals-skillsbench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":51.274,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.827586206896555,"participants":30,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 495.558. Standardfehler: 4.6. Kosten je Aufgabe in USD: 0.397123. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 495.558. Standard error: 4.6. Cost per task in USD: 0.397123. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":50,"measurements":[{"observationId":"vals-snap:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":62.923,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":39,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 922.395. Standardfehler: 1.256. Kosten je Aufgabe in USD: 0.460453. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 922.395. Standard error: 1.256. Cost per task in USD: 0.460453. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":56.36363636363637,"measurements":[{"observationId":"vals-tax-agent-bench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":25.17,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.36363636363637,"participants":56,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1244.635. Standardfehler: 3.058. Kosten je Aufgabe in USD: 0.719152. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 384000. Latency in seconds: 1244.635. Standard error: 3.058. Cost per task in USD: 0.719152. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":43.421052631578945,"measurements":[{"observationId":"vals-tax-eval-v2:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.077,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.421052631578945,"participants":77,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 189.695. Standardfehler: 0.877. Kosten je Aufgabe in USD: 0.033187. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 189.695. Standard error: 0.877. Cost per task in USD: 0.033187. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":48.611111111111114,"measurements":[{"observationId":"vals-vibe-code-bench:overall:deepseek/deepseek-v4-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":49.931,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.611111111111114,"participants":73,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4-pro. Name des Quellmodells: DeepSeek-V4-Pro. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 3418.92. Standardfehler: 4.767. Kosten je Aufgabe in USD: 2.205991. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4-pro. Source model name: DeepSeek-V4-Pro. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 256000. Latency in seconds: 3418.92. Standard error: 4.767. Cost per task in USD: 2.205991. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":33,"totalFamilies":71,"sourceCount":3,"coverage":0.40051020408163274,"lowerBound":16.862381484330253,"upperBound":76.81136107616697,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":2,"coverage":0.40000000000000013,"lowerBound":20.36555322341327,"upperBound":80.36555322341326,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":45.235038816553576,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":29.079667810641585,"upperBound":64.7939535249273,"provisional":true,"configurationsDiffer":true},"math":{"score":16.58896658896659,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":9.953379953379955,"upperBound":49.95337995337995,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":51.361465995612335,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":51.361465995612335,"upperBound":51.361465995612335,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":2,"coverage":0.3333333333333333,"lowerBound":21.82506307821699,"upperBound":88.49172974488366,"provisional":true,"configurationsDiffer":true},"finance":{"score":42.265839921504515,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":30.189885658217506,"upperBound":58.76131422964609,"provisional":true,"configurationsDiffer":true},"legal":{"score":40.673252066694694,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":30.504939050021022,"upperBound":55.50493905002102,"provisional":true,"configurationsDiffer":true},"health":{"score":26.690079016681302,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":17.793386011120866,"upperBound":51.1267193444542,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":25,"upperBound":75,"provisional":true,"configurationsDiffer":true}}},{"modelId":"deepseek-v4.1-flash","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":35.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":35,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 35,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 35.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1420.2,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":96.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":84,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 84,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 84.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":68.89097769674058,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 68,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 68.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":80,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":44.0335333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 44,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 44.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":57.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":12.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 12,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 12.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1600,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":77.14285714285714,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":46.9161958568738,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":77.14285714285714,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 46,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 46.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":49.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":-5.3,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":39.2493049119555,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 39,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 39.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":61.111111111111114,"measurements":[{"observationId":"lmarena:webdev:overall:deepseek-v4.1-flash-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1618.5134057265923,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":61.111111111111114,"participants":37,"methodology":{"de":"Arena-Bewertung aus 3.544 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/deepseek-v4.1-flash-max. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-v4.1-flash-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 3,544 published pairwise votes.. Source model ID: lmarena/deepseek-v4.1-flash-max. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: deepseek-v4.1-flash-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 3.544 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 3,544 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 3.544 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 3,544 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":50,"measurements":[{"observationId":"lmarena:overall:deepseek-v4.1-flash-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1474.5790515078297,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":50,"participants":37,"methodology":{"de":"Arena-Bewertung aus 12.000 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/deepseek-v4.1-flash-max. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-v4.1-flash-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 12,000 published pairwise votes.. Source model ID: lmarena/deepseek-v4.1-flash-max. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: deepseek-v4.1-flash-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 12.000 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 12,000 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 12.000 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 12,000 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"lmarena:vision_style_control:overall:deepseek-v4.1-flash-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1260.3175998274655,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":26.666666666666668,"participants":31,"methodology":{"de":"Arena-Bewertung aus 3.559 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/deepseek-v4.1-flash-max. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: deepseek-v4.1-flash-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 3,559 published pairwise votes.. Source model ID: lmarena/deepseek-v4.1-flash-max. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: deepseek-v4.1-flash-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 3.559 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 3,559 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 3.559 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 3,559 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":53.57142857142857,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":76.9942196531792,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.57142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 77,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 77.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":73.3139890761842,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":26.7676767676768,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.3170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 26,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 26.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":74.532,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.32142857142857,"participants":57,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 604.506. Standardfehler: 1.633. Kosten je Aufgabe in USD: 0.098877. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 604.506. Standard error: 1.633. Cost per task in USD: 0.098877. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":19.697,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.2972972972973,"participants":38,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 2902.323. Standardfehler: 1.75. Kosten je Aufgabe in USD: 0.499473. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 2902.323. Standard error: 1.75. Cost per task in USD: 0.499473. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":26.332398316970547,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:dbe7c625-3100-4463-b479-a228c41f75dd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":9.047619047619051,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.82608695652174,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Modellstand: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 9,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/dbe7c625-3100-4463-b479-a228c41f75dd/deepseek-v4-1-flash/deepseek-v4-1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Model snapshot: DeepSeek V4.1 Flash (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 9.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: DeepSeek V4.1 Flash (Max) (Release deepseek-v4-1-flash, Zeile deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: DeepSeek V4.1 Flash (Max) (release deepseek-v4-1-flash, row deepseek-v4-1-flash, UUID dbe7c625-3100-4463-b479-a228c41f75dd, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.838709677419355,"participants":32,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 10621.573. Standardfehler: 0. Kosten je Aufgabe in USD: 0.473154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 10621.573. Standard error: 0. Cost per task in USD: 0.473154. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":36.8421052631579,"measurements":[{"observationId":"vals-biomysterybench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":67.778,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.8421052631579,"participants":20,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 890.067. Standardfehler: 1.111. Kosten je Aufgabe in USD: 0.233183. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 890.067. Standard error: 1.111. Cost per task in USD: 0.233183. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: Terminus 2. Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: Terminus 2. Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":79.03225806451613,"measurements":[{"observationId":"vals-code-migration:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":45.623,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.03225806451613,"participants":63,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 6325.865. Standardfehler: 4.285. Kosten je Aufgabe in USD: 0.944694. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 6325.865. Standard error: 4.285. Cost per task in USD: 0.944694. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"vals-cyberbench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":73.69,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":34,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 830.108. Standardfehler: 5.476. Kosten je Aufgabe in USD: 0.069681. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 830.108. Standard error: 5.476. Cost per task in USD: 0.069681. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":45.76271186440678,"measurements":[{"observationId":"vals-emb:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":57.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.76271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 780.458. Standardfehler: 3.227. Kosten je Aufgabe in USD: 0.234152. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 780.458. Standard error: 3.227. Cost per task in USD: 0.234152. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":61.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.481,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 364.376. Standardfehler: 0.39. Kosten je Aufgabe in USD: 0.207465. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 364.376. Standard error: 0.39. Cost per task in USD: 0.207465. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":15.151515151515152,"measurements":[{"observationId":"vals-ioi:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":40.278,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.151515151515152,"participants":34,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1109.577. Standardfehler: 2.557. Kosten je Aufgabe in USD: 0.24255. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 1109.577. Standard error: 2.557. Cost per task in USD: 0.24255. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":70.49180327868852,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":6.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.49180327868852,"participants":62,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 579.44. Standardfehler: 1.832. Kosten je Aufgabe in USD: 0.173471. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 579.44. Standard error: 1.832. Cost per task in USD: 0.173471. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":51.282051282051285,"measurements":[{"observationId":"vals-legal-bench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.277,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.282051282051285,"participants":79,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 6.533. Standardfehler: 0.458. Kosten je Aufgabe in USD: 0.001419. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 6.533. Standard error: 0.458. Cost per task in USD: 0.001419. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":70.83333333333333,"measurements":[{"observationId":"vals-legal-research:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":41.346,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.83333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 701.783. Standardfehler: 3.423. Kosten je Aufgabe in USD: 0.24793. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 701.783. Standard error: 3.423. Cost per task in USD: 0.24793. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":38.80597014925373,"measurements":[{"observationId":"vals-medcode:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":41.173,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.80597014925373,"participants":68,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 35.126. Standardfehler: 2.042. Kosten je Aufgabe in USD: 0.01245. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 35.126. Standard error: 2.042. Cost per task in USD: 0.01245. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":72.05882352941177,"measurements":[{"observationId":"vals-medscribe:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":85.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.05882352941177,"participants":69,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 53.826. Standardfehler: 1.918. Kosten je Aufgabe in USD: 0.015397. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 53.826. Standard error: 1.918. Cost per task in USD: 0.015397. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":20,"measurements":[{"observationId":"vals-mysterymechanism:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":21.171,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20,"participants":21,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 484.327. Standardfehler: 2.748. Kosten je Aufgabe in USD: 0.164089. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 484.327. Standard error: 2.748. Cost per task in USD: 0.164089. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 3709.707. Standardfehler: 0.5. Kosten je Aufgabe in USD: 0.896935. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 3709.707. Standard error: 0.5. Cost per task in USD: 0.896935. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":48.717948717948715,"measurements":[{"observationId":"vals-proof-bench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":54,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.717948717948715,"participants":40,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 546.595. Standardfehler: 5.009. Kosten je Aufgabe in USD: 0.126437. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 546.595. Standard error: 5.009. Cost per task in USD: 0.126437. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":58.62068965517241,"measurements":[{"observationId":"vals-sage:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":47.88,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.62068965517241,"participants":59,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 35.776. Standardfehler: 3.438. Kosten je Aufgabe in USD: 0.01219. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 35.776. Standard error: 3.438. Cost per task in USD: 0.01219. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"vals-skillsbench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":69.802,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":30,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 223.451. Standardfehler: 3.879. Kosten je Aufgabe in USD: 0.085117. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 223.451. Standard error: 3.879. Cost per task in USD: 0.085117. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":57.89473684210526,"measurements":[{"observationId":"vals-snap:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":64.276,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.89473684210526,"participants":39,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 598.96. Standardfehler: 1.246. Kosten je Aufgabe in USD: 0.073234. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 598.96. Standard error: 1.246. Cost per task in USD: 0.073234. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":20.833333333333332,"measurements":[{"observationId":"vals-sre-bench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0.763,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.833333333333332,"participants":25,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 3663.235. Standardfehler: 0.539. Kosten je Aufgabe in USD: 0.548583. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 3663.235. Standard error: 0.539. Cost per task in USD: 0.548583. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"vals-swebench-multimodal:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":46.458,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.23076923076923,"participants":14,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1164.341. Standardfehler: 2.279. Kosten je Aufgabe in USD: 0.225505. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 1164.341. Standard error: 2.279. Cost per task in USD: 0.225505. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":60,"measurements":[{"observationId":"vals-tax-agent-bench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":26.63,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60,"participants":56,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 349.076. Standardfehler: 3.141. Kosten je Aufgabe in USD: 0.134662. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000. Latency in seconds: 349.076. Standard error: 3.141. Cost per task in USD: 0.134662. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 384000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":42.857142857142854,"measurements":[{"observationId":"vals-vcb-1-100:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":16.378,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":22,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 1701.009. Standardfehler: 3.194. Kosten je Aufgabe in USD: 0.688342. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 1701.009. Standard error: 3.194. Cost per task in USD: 0.688342. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"vals-vibe-code-bench:overall:deepseek/deepseek-v4.1-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":84.739,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":73,"methodology":{"de":"Kennung des Quellmodells: deepseek/deepseek-v4.1-flash. Name des Quellmodells: DeepSeek-V4.1-Flash. Anbieter laut Quelle: DeepSeek. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000. Latenz in Sekunden: 928.101. Standardfehler: 2.891. Kosten je Aufgabe in USD: 0.407445. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: deepseek/deepseek-v4.1-flash. Source model name: DeepSeek-V4.1-Flash. Provider reported by source: DeepSeek. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000. Latency in seconds: 928.101. Standard error: 2.891. Cost per task in USD: 0.407445. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 384000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 384000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":59.28202264980729,"coveredFamilies":38,"totalFamilies":71,"sourceCount":3,"coverage":0.504733560090703,"lowerBound":29.92162634141492,"upperBound":79.44827033234462,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":58.81501022998289,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.6555555555555557,"lowerBound":38.556506706322125,"upperBound":73.00095115076655,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":26.56794425087108,"upperBound":90.85365853658537,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":9.743589743589745,"upperBound":89.74358974358975,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.463414634146343,"upperBound":83.13008130081302,"provisional":true,"configurationsDiffer":true},"agentic":{"score":79.14404914725839,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":39.57202457362919,"upperBound":89.5720245736292,"provisional":true,"configurationsDiffer":true},"general":{"score":31.918859649122805,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":25.535087719298247,"upperBound":45.53508771929825,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":23.918482647296205,"upperBound":81.06133979015334,"provisional":true,"configurationsDiffer":true},"legal":{"score":64.20239596469104,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":48.15179697351828,"upperBound":73.15179697351829,"provisional":true,"configurationsDiffer":true},"health":{"score":55.432396839332746,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":36.95493122622183,"upperBound":70.28826455955516,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":81.81818181818181,"upperBound":81.81818181818181,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":13.363095238095237,"upperBound":75.86309523809524,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":28.94736842105263,"upperBound":78.94736842105263,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":29.310344827586206,"upperBound":79.3103448275862,"provisional":true,"configurationsDiffer":true}}},{"modelId":"ember-1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ernie-4.5-vl-424b-a47b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"falcon-emirati-7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"fugu-max","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"fugu-ultra-v2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gemini-2.5-flash","contributions":[{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":25,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gemini-2.5-flash","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":46.7,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":25,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gemini-2.5-flash. Name des Quellmodells: Gemini 2.5 Flash. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-ec0350e989af. Prüfsumme der Quelldaten: ec0350e989af492497edfa7fb858ea21c4693c375e9212b1ad488dc3f8eb81f8. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gemini-2.5-flash. Source model name: Gemini 2.5 Flash. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-ec0350e989af. Source payload hash: ec0350e989af492497edfa7fb858ea21c4693c375e9212b1ad488dc3f8eb81f8. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":12.5,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gemini-2.5-flash","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":59,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":12.5,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gemini-2.5-flash. Name des Quellmodells: Gemini 2.5 Flash. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-781c5011daf3. Prüfsumme der Quelldaten: 781c5011daf31b3367fa81ab29685b1d3b7b0e7f082a01f9888879c8a7223b33. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gemini-2.5-flash. Source model name: Gemini 2.5 Flash. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-781c5011daf3. Source payload hash: 781c5011daf31b3367fa81ab29685b1d3b7b0e7f082a01f9888879c8a7223b33. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"curated:scale-prbench-finance-gemini-2.5-flash","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":38.41,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":13.333333333333334,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-2.5-flash. Name des Quellmodells: Gemini 2.5 Flash. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 38,41 % ± 0,62. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-88c18b250ab8. Prüfsumme der Quelldaten: 88c18b250ab815175025671db93668dc330c2e53f517c9e8a1de37d51f27b988. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-2.5-flash. Source model name: Gemini 2.5 Flash. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 38.41% ± 0.62. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-88c18b250ab8. Source payload hash: 88c18b250ab815175025671db93668dc330c2e53f517c9e8a1de37d51f27b988. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"curated:scale-prbench-legal-gemini-2.5-flash","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":41.02,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":26.666666666666668,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-2.5-flash. Name des Quellmodells: Gemini 2.5 Flash. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 41,02 % ± 0,61. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-4e04093a101b. Prüfsumme der Quelldaten: 4e04093a101b1d3499b6333e4267cf0de410540678436d292d53e2bdc6fdb3ca. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-2.5-flash. Source model name: Gemini 2.5 Flash. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 41.02% ± 0.61. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-4e04093a101b. Source payload hash: 4e04093a101b1d3499b6333e4267cf0de410540678436d292d53e2bdc6fdb3ca. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"curated:scale-visual-toolbench-gemini-2.5-flash","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":4.69,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":22.22222222222222,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/gemini-2.5-flash. Name des Quellmodells: Gemini 2.5 Flash. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 4,69 % ± 0,13. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-d08ed1b114e6. Prüfsumme der Quelldaten: d08ed1b114e6589943992e9b9a599fec181130395dc31d8682e7dc5ca15a7130. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/gemini-2.5-flash. Source model name: Gemini 2.5 Flash. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 4.69% ± 0.13. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-d08ed1b114e6. Source payload hash: d08ed1b114e6589943992e9b9a599fec181130395dc31d8682e7dc5ca15a7130. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":5,"totalFamilies":71,"sourceCount":2,"coverage":0.05782312925170068,"lowerBound":1.2322845804988662,"upperBound":95.44997165532881,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-08-26","areas":{"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":3.125,"upperBound":86.45833333333334,"provisional":true,"configurationsDiffer":false},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":1.9047619047619047,"upperBound":87.61904761904762,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":6.666666666666667,"upperBound":81.66666666666667,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":5.555555555555555,"upperBound":80.55555555555556,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-2.5-flash-lite","contributions":[{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":1.492537313432836,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-2.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":27.115,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.492537313432836,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-flash-lite. Name des Quellmodells: Gemini 2.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 5.746. Standardfehler: 1.843. Kosten je Aufgabe in USD: 0.001342. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-flash-lite. Source model name: Gemini 2.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 5.746. Standard error: 1.843. Cost per task in USD: 0.001342. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":17.647058823529413,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-2.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":72.832,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.647058823529413,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-flash-lite. Name des Quellmodells: Gemini 2.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 4.847. Standardfehler: 1.982. Kosten je Aufgabe in USD: 0.001211. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-flash-lite. Source model name: Gemini 2.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 4.847. Standard error: 1.982. Cost per task in USD: 0.001211. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":3.4482758620689653,"measurements":[{"observationId":"vals-sage:overall:google/gemini-2.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":27.533,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.4482758620689653,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-flash-lite. Name des Quellmodells: Gemini 2.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 13.781. Standardfehler: 3.124. Kosten je Aufgabe in USD: 0.000929. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-flash-lite. Source model name: Gemini 2.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 13.781. Standard error: 3.124. Cost per task in USD: 0.000929. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":3,"totalFamilies":71,"sourceCount":1,"coverage":0.08333333333333334,"lowerBound":0.5788573792872785,"upperBound":92.24552404595394,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"health":{"score":9.569798068481123,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":6.379865378987416,"upperBound":39.713198712320754,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":1.7241379310344827,"upperBound":51.724137931034484,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-2.5-pro","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":301.88,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":69,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 69,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 69.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":2.174946419340874,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 2,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 2.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":10.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 10,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 10.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":455.95,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":42.028985507246375,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":48.707482993197296,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.028985507246375,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 48,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 48.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-16.35,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":37.5,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gemini-2.5-pro","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":52.5,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":37.5,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-5c16f4ffaaa5. Prüfsumme der Quelldaten: 5c16f4ffaaa5ac8a5c682da01aa3fab9655de1625348891876a58ad2be1d678f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-5c16f4ffaaa5. Source payload hash: 5c16f4ffaaa5ac8a5c682da01aa3fab9655de1625348891876a58ad2be1d678f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":37.5,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gemini-2.5-pro","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":71.8,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":37.5,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-c647c71a1881. Prüfsumme der Quelldaten: c647c71a188169cdd3c4727d53414ec2e2f40c9bc60f3918df963c30270089e4. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-c647c71a1881. Source payload hash: c647c71a188169cdd3c4727d53414ec2e2f40c9bc60f3918df963c30270089e4. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":22.5208526413346,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 22,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 22.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":42.857142857142854,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":74.9132947976879,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.857142857142854,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 74,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 74.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"curated:scale-multinrc-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":45.12,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 45,12 % ± 3. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-28c12d03ebe3. Prüfsumme der Quelldaten: 28c12d03ebe3101b8d86bff6e5892f3ee72da8548781a307dcb96460002dcd0c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 45.12% ± 3. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-28c12d03ebe3. Source payload hash: 28c12d03ebe3101b8d86bff6e5892f3ee72da8548781a307dcb96460002dcd0c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":20,"measurements":[{"observationId":"curated:scale-prbench-finance-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":38.92,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":20,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 38,92 % ± 0,48. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-ffcfbc6e9581. Prüfsumme der Quelldaten: ffcfbc6e9581d4bd432f101f0f9a9bfe463461b774ac00d936de7274160b0665. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 38.92% ± 0.48. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-ffcfbc6e9581. Source payload hash: ffcfbc6e9581d4bd432f101f0f9a9bfe463461b774ac00d936de7274160b0665. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"curated:scale-prbench-legal-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":41.43,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 41,43 % ± 0,2. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-11f363ace277. Prüfsumme der Quelldaten: 11f363ace2777f0cdad654b5ad414bb85e2cc344edd4c1aeec66b22d688ad817. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 41.43% ± 0.2. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-11f363ace277. Source payload hash: 11f363ace2777f0cdad654b5ad414bb85e2cc344edd4c1aeec66b22d688ad817. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":6.779661016949152,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":54.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.779661016949152,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 207.791. Standardfehler: 2.23. Kosten je Aufgabe in USD: 0.351578. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 207.791. Standard error: 2.23. Cost per task in USD: 0.351578. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":20.107433217189318,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:27202e5f-c82d-4710-92e9-4317877d4883","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/27202e5f-c82d-4710-92e9-4317877d4883/gemini-2-5-pro/gemini-2-5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: Gemini 2.5 Pro. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 2.5 Pro (Release gemini-2-5-pro, Zeile gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 2.5 Pro (release gemini-2-5-pro, row gemini-2-5-pro, UUID 27202e5f-c82d-4710-92e9-4317877d4883, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":30.337,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.38095238095238,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 796.526. Standardfehler: 4.901. Kosten je Aufgabe in USD: 0.472003. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 796.526. Standard error: 4.901. Cost per task in USD: 0.472003. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"curated:scale-tutorbench-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":55.65,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":88.88888888888889,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: gemini-2.5-pro-preview-06-05. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 55,65 % ± 1,11. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-ec5879ea85c2. Prüfsumme der Quelldaten: ec5879ea85c27f00c900adfeaf220de4cc2324875c7509c9aafab1af046ae9d5. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: gemini-2.5-pro-preview-06-05. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 55.65% ± 1.11. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-ec5879ea85c2. Source payload hash: ec5879ea85c27f00c900adfeaf220de4cc2324875c7509c9aafab1af046ae9d5. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":41.21621621621622,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.21621621621622,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 27.425. Standardfehler: 0.961. Kosten je Aufgabe in USD: 0.09535. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 27.425. Standard error: 0.961. Cost per task in USD: 0.09535. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":82.08955223880596,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":50.59,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.08955223880596,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 27.415. Standardfehler: 2.113. Kosten je Aufgabe in USD: 0.015389. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 27.415. Standard error: 2.113. Cost per task in USD: 0.015389. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":20.58823529411765,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":73.552,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.58823529411765,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 35.909. Standardfehler: 1.91. Kosten je Aufgabe in USD: 0.046379. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 35.909. Standard error: 1.91. Cost per task in USD: 0.046379. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":86.53846153846153,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.918,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.53846153846153,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 24.008. Standardfehler: 0.907. Kosten je Aufgabe in USD: 0.004794. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 24.008. Standard error: 0.907. Cost per task in USD: 0.004794. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":37.5,"measurements":[{"observationId":"vals-poker-agent:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1032.5962096202413,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.5,"participants":9,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 19.79321908170333. Standardfehler: 0. Kosten je Aufgabe in USD: 0.02165833147163121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 19.79321908170333. Standard error: 0. Cost per task in USD: 0.02165833147163121. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":31.03448275862069,"measurements":[{"observationId":"vals-sage:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":41.916,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.03448275862069,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 52.934. Standardfehler: 3.411. Kosten je Aufgabe in USD: 0.008806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 52.934. Standard error: 3.411. Cost per task in USD: 0.008806. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":4.166666666666667,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":0.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.166666666666667,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2097.144. Standardfehler: 0.4. Kosten je Aufgabe in USD: 1.223831. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2097.144. Standard error: 0.4. Cost per task in USD: 1.223831. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-vista-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":54.63,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Modellstand: gemini-2.5-pro-preview-06-05. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 54,63 % ± 0,55. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-521700020ded. Prüfsumme der Quelldaten: 521700020deda7eff22b3228caecf8b09a79628e026db33656f233c07198cd16. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Model snapshot: gemini-2.5-pro-preview-06-05. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 54.63% ± 0.55. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-521700020ded. Source payload hash: 521700020deda7eff22b3228caecf8b09a79628e026db33656f233c07198cd16. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"curated:scale-visual-toolbench-gemini-2.5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":11.75,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":55.55555555555556,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/gemini-2.5-pro. Name des Quellmodells: Gemini 2.5 Pro. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 11,75 % ± 1,14. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-a155ac32bbe9. Prüfsumme der Quelldaten: a155ac32bbe96c3d39f82628f8d93daa034f220a28e4a71debec7e7cadd140c8. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/gemini-2.5-pro. Source model name: Gemini 2.5 Pro. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 11.75% ± 1.14. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-a155ac32bbe9. Source payload hash: a155ac32bbe96c3d39f82628f8d93daa034f220a28e4a71debec7e7cadd140c8. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":26,"totalFamilies":71,"sourceCount":4,"coverage":0.33200113378684804,"lowerBound":15.829367376352373,"upperBound":82.62925399766758,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":2,"coverage":0.14444444444444451,"lowerBound":1.3974301768844974,"upperBound":86.95298573244006,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":31.311371555273993,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5,"lowerBound":15.655685777636997,"upperBound":65.65568577763699,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":13.414634146341463,"upperBound":80.08130081300814,"provisional":true,"configurationsDiffer":true},"agentic":{"score":29.72560975609756,"coveredFamilies":6,"totalFamilies":12,"sourceCount":3,"coverage":0.5,"lowerBound":14.86280487804878,"upperBound":64.86280487804878,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":8.405797101449275,"upperBound":88.40579710144928,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":2,"coverage":0.4285714285714286,"lowerBound":21.107811107811106,"upperBound":78.25066825066824,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":8.333333333333334,"upperBound":83.33333333333333,"provisional":true,"configurationsDiffer":true},"health":{"score":51.3388937664618,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":34.2259291776412,"upperBound":67.55926251097453,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":70.7936507936508,"coveredFamilies":3,"totalFamilies":4,"sourceCount":2,"coverage":0.625,"lowerBound":44.24603174603175,"upperBound":81.74603174603175,"provisional":true,"configurationsDiffer":true},"education":{"score":59.96168582375479,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":59.96168582375479,"upperBound":59.96168582375479,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3-flash","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:7c73c3be-7f51-4d14-bec8-d5789488df25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Reasoning","retrievedAt":"2026-10-19","value":27.728613569321496,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3 Flash Preview (Reasoning) (Release gemini-3-flash-preview, Zeile gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/7c73c3be-7f51-4d14-bec8-d5789488df25/gemini-3-flash-preview/gemini-3-flash-reasoning. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3 Flash Preview (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 27,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3 Flash Preview (Reasoning) (release gemini-3-flash-preview, row gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/7c73c3be-7f51-4d14-bec8-d5789488df25/gemini-3-flash-preview/gemini-3-flash-reasoning. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Model snapshot: Gemini 3 Flash Preview (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 27.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3 Flash Preview (Reasoning) (Release gemini-3-flash-preview, Zeile gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemini 3 Flash Preview (Reasoning) (release gemini-3-flash-preview, row gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3 Flash Preview (Reasoning) (Release gemini-3-flash-preview, Zeile gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemini 3 Flash Preview (Reasoning) (release gemini-3-flash-preview, row gemini-3-flash-reasoning, UUID 7c73c3be-7f51-4d14-bec8-d5789488df25, reasoning=true, source reasoning-token configuration budget=0, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":63.1578947368421,"measurements":[{"observationId":"vals-gpqa:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":87.879,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.1578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 83.84. Standardfehler: 1.638. Kosten je Aufgabe in USD: 0.044346. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 83.84. Standard error: 1.638. Cost per task in USD: 0.044346. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":74.46808510638297,"measurements":[{"observationId":"vals-mmmu:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":87.63,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.46808510638297,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 27.856. Standardfehler: 0.792. Kosten je Aufgabe in USD: 0.014422. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 27.856. Standard error: 0.792. Cost per task in USD: 0.014422. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":45.76271186440678,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.76271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 285.223. Standardfehler: 1.938. Kosten je Aufgabe in USD: 0.280231. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 285.223. Standard error: 1.938. Cost per task in USD: 0.280231. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":49.70238095238095,"measurements":[{"observationId":"vals-terminal-bench-2:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":51.685,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.04761904761905,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 456.071. Standardfehler: 5.327. Kosten je Aufgabe in USD: 0.168702. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 456.071. Standard error: 5.327. Cost per task in USD: 0.168702. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":53.933,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.357142857142858,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 376.158. Standardfehler: 1.297. Kosten je Aufgabe in USD: 0.154312. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 376.158. Standard error: 1.297. Cost per task in USD: 0.154312. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":12.903225806451612,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":6.391,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.903225806451612,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 284.234. Standardfehler: 1.093. Kosten je Aufgabe in USD: 0.340087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 284.234. Standard error: 1.093. Cost per task in USD: 0.340087. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":82.43243243243244,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.43243243243244,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 11.279. Standardfehler: 0.93. Kosten je Aufgabe in USD: 0.047154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 11.279. Standard error: 0.93. Cost per task in USD: 0.047154. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":11.864406779661017,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":26.017,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.864406779661017,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 328.69. Standardfehler: 2.912. Kosten je Aufgabe in USD: 0.453183. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 328.69. Standard error: 2.912. Cost per task in USD: 0.453183. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":25,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":42.551,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 246.543. Standardfehler: 0.434. Kosten je Aufgabe in USD: 0.570657. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 246.543. Standard error: 0.434. Cost per task in USD: 0.570657. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.591,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.42857142857143,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 91.931. Standardfehler: 1.004. Kosten je Aufgabe in USD: 0.050525. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 91.931. Standard error: 1.004. Cost per task in USD: 0.050525. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 202.513. Standardfehler: 0. Kosten je Aufgabe in USD: 0.264375. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 202.513. Standard error: 0. Cost per task in USD: 0.264375. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":89.74358974358974,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.858,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.74358974358974,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.673. Standardfehler: 0.356. Kosten je Aufgabe in USD: 0.002067. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.673. Standard error: 0.356. Cost per task in USD: 0.002067. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":28.333333333333332,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":18.269,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 187.7. Standardfehler: 2.686. Kosten je Aufgabe in USD: 0.135395. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 187.7. Standard error: 2.686. Cost per task in USD: 0.135395. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":95.5223880597015,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":55.92,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.5223880597015,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 44.149. Standardfehler: 2.112. Kosten je Aufgabe in USD: 0.006187. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 44.149. Standard error: 2.112. Cost per task in USD: 0.006187. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":82.05128205128206,"measurements":[{"observationId":"vals-medqa:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":95.808,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.05128205128206,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 26.324. Standardfehler: 0.183. Kosten je Aufgabe in USD: 0.012699. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 26.324. Standard error: 0.183. Cost per task in USD: 0.012699. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":11.764705882352942,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":69.917,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.764705882352942,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 23.491. Standardfehler: 1.899. Kosten je Aufgabe in USD: 0.014379. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 23.491. Standard error: 1.899. Cost per task in USD: 0.014379. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":93.10344827586206,"measurements":[{"observationId":"vals-mgsm:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":93.309,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.10344827586206,"participants":30,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 14.02. Standardfehler: 1.456. Kosten je Aufgabe in USD: 0.007116. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 14.02. Standard error: 1.456. Cost per task in USD: 0.007116. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":80.51948051948052,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":88.592,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.51948051948052,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 36.249. Standardfehler: 0.315. Kosten je Aufgabe in USD: 0.019364. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 36.249. Standard error: 0.315. Cost per task in USD: 0.019364. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":82.6923076923077,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.72,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.6923076923077,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.838. Standardfehler: 0.914. Kosten je Aufgabe in USD: 0.007477. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.838. Standard error: 0.914. Cost per task in USD: 0.007477. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":87.5,"measurements":[{"observationId":"vals-poker-agent:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1100.2125195039544,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.5,"participants":9,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 7.7072606111437345. Standardfehler: 0. Kosten je Aufgabe in USD: 0.00872674530884417. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 7.7072606111437345. Standard error: 0. Cost per task in USD: 0.00872674530884417. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 418.92. Standardfehler: 0. Kosten je Aufgabe in USD: 0.439965. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 418.92. Standard error: 0. Cost per task in USD: 0.439965. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":87.93103448275862,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":51.849,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.93103448275862,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 42.697. Standardfehler: 3.404. Kosten je Aufgabe in USD: 0.018229. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 42.697. Standard error: 3.404. Cost per task in USD: 0.018229. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":25.454545454545453,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":11.843,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.454545454545453,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 100.623. Standardfehler: 2.324. Kosten je Aufgabe in USD: 0.100699. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 100.623. Standard error: 2.324. Cost per task in USD: 0.100699. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":59.21052631578947,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.876,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.21052631578947,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.001. Standardfehler: 0.857. Kosten je Aufgabe in USD: 0.005764. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.001. Standard error: 0.857. Cost per task in USD: 0.005764. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":26.38888888888889,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3-flash-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":20.204,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.38888888888889,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3-flash-preview. Name des Quellmodells: Gemini 3 Flash Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 806.66. Standardfehler: 3.946. Kosten je Aufgabe in USD: 0.942067. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3-flash-preview. Source model name: Gemini 3 Flash Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 806.66. Standard error: 3.946. Cost per task in USD: 0.942067. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":25,"totalFamilies":71,"sourceCount":2,"coverage":0.3089852607709751,"lowerBound":18.75676207028714,"upperBound":87.85823599318962,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":1,"coverage":0.3222222222222223,"lowerBound":11.64395321946287,"upperBound":79.42173099724064,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":4.511278195488722,"upperBound":97.36842105263159,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":18.620689655172413,"upperBound":98.62068965517241,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":26.83982683982684,"upperBound":93.50649350649351,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":2,"coverage":0.16666666666666666,"lowerBound":11.458333333333334,"upperBound":94.79166666666667,"provisional":true,"configurationsDiffer":true},"finance":{"score":47.775703112456014,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":40.95060266781944,"upperBound":55.23631695353373,"provisional":true,"configurationsDiffer":true},"legal":{"score":42.910886927280366,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":32.183165195460276,"upperBound":57.183165195460276,"provisional":true,"configurationsDiffer":true},"health":{"score":63.112791997778835,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":63.112791997778835,"upperBound":63.112791997778835,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":9.308510638297872,"upperBound":96.80851063829788,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":43.96551724137931,"upperBound":93.9655172413793,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.1-flash-lite","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":3.9473684210526314,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":8.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.9473684210526314,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 8,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 8.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":207.25,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":74.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 74,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 74.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":19.23076923076923,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":12.168141592920401,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.23076923076923,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 12,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 12.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":6.796182033119545,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 6,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 6.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":37.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":7.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 7,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 7.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":430.83,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":94.20289855072464,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":77.2108843537415,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":94.20289855072464,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 77,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 77.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-16.416666666666668,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":38.1578947368421,"measurements":[{"observationId":"vals-gpqa:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":81.06,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.1578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 16.464. Standardfehler: 1.967. Kosten je Aufgabe in USD: 0.002298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 16.464. Standard error: 1.967. Cost per task in USD: 0.002298. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":17.191844300278,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 17,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 17.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":45.38373860182371,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":75.5491329479769,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.214285714285715,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 75,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 75.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":82.486,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.5531914893617,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.448. Standardfehler: 0.914. Kosten je Aufgabe in USD: 0.003187. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.448. Standard error: 0.914. Cost per task in USD: 0.003187. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":11.864406779661017,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":62.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.864406779661017,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 132.936. Standardfehler: 2.164. Kosten je Aufgabe in USD: 0.105389. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 132.936. Standard error: 2.164. Cost per task in USD: 0.105389. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":28.202671312427412,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:598190f8-dc9c-4fea-a7ea-4b81c402ab18","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/598190f8-dc9c-4fea-a7ea-4b81c402ab18/gemini-3-1-flash-lite/gemini-3-1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.1 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Flash-Lite (Release gemini-3-1-flash-lite, Zeile gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Flash-Lite (release gemini-3-1-flash-lite, row gemini-3-1-flash-lite-preview, UUID 598190f8-dc9c-4fea-a7ea-4b81c402ab18, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":24.719,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.666666666666668,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 667.879. Standardfehler: 4.599. Kosten je Aufgabe in USD: 0.065496. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 667.879. Standard error: 4.599. Cost per task in USD: 0.065496. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":34.082,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.7857142857142858,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 278.342. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.084894. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 278.342. Standard error: 0.991. Cost per task in USD: 0.084894. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"curated:scale-tutorbench-gemini-3.1-flash-lite","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":51.5,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":22.22222222222222,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gemini-3.1-flash-lite. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 51,5 % ± 1,04. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-b9d2857e0533. Prüfsumme der Quelldaten: b9d2857e0533b70e6dacdf8d12a36816aa5704941213476aa6e699f087ec4ff0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gemini-3.1-flash-lite. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 51.5% ± 1.04. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-b9d2857e0533. Source payload hash: b9d2857e0533b70e6dacdf8d12a36816aa5704941213476aa6e699f087ec4ff0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":6.451612903225806,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":4.609,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.451612903225806,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 92.63. Standardfehler: 1.069. Kosten je Aufgabe in USD: 0.053028. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 92.63. Standard error: 1.069. Cost per task in USD: 0.053028. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":29.72972972972973,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":59.363,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.72972972972973,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.03. Standardfehler: 0.967. Kosten je Aufgabe in USD: 0.022927. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.03. Standard error: 0.967. Cost per task in USD: 0.022927. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":8.629,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 67.007. Standardfehler: 1.088. Kosten je Aufgabe in USD: 0.069424. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 67.007. Standard error: 1.088. Cost per task in USD: 0.069424. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":29.988,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.666666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 54.778. Standardfehler: 0.643. Kosten je Aufgabe in USD: 0.142482. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 54.778. Standard error: 0.643. Cost per task in USD: 0.142482. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":32.467532467532465,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":80.116,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.467532467532465,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 30.749. Standardfehler: 1.083. Kosten je Aufgabe in USD: 0.00435. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 30.749. Standard error: 1.083. Cost per task in USD: 0.00435. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 60.554. Standardfehler: 0. Kosten je Aufgabe in USD: 0.084048. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 60.554. Standard error: 0. Cost per task in USD: 0.084048. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":58.97435897435897,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.764,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.97435897435897,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2.704. Standardfehler: 0.406. Kosten je Aufgabe in USD: 0.000295. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2.704. Standard error: 0.406. Cost per task in USD: 0.000295. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":3.3333333333333335,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":3.365,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.3333333333333335,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 20.403. Standardfehler: 1.253. Kosten je Aufgabe in USD: 0.013607. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 20.403. Standard error: 1.253. Cost per task in USD: 0.013607. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":65.67164179104478,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":47.602,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.67164179104478,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 8.55. Standardfehler: 2.071. Kosten je Aufgabe in USD: 0.002029. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 8.55. Standard error: 2.071. Cost per task in USD: 0.002029. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":5.882352941176471,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":63.902,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.882352941176471,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.793. Standardfehler: 1.823. Kosten je Aufgabe in USD: 0.002195. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 16.793. Standard error: 1.823. Cost per task in USD: 0.002195. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":59.74025974025974,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.24,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.74025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 14.511. Standardfehler: 0.34. Kosten je Aufgabe in USD: 0.001052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 14.511. Standard error: 0.34. Cost per task in USD: 0.001052. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.044,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.23076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 5.368. Standardfehler: 0.912. Kosten je Aufgabe in USD: 0.00284. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 5.368. Standard error: 0.912. Cost per task in USD: 0.00284. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 176.246. Standardfehler: 0. Kosten je Aufgabe in USD: 0.063906. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 176.246. Standard error: 0. Cost per task in USD: 0.063906. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":72.41379310344827,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":49.54,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.41379310344827,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.257. Standardfehler: 3.484. Kosten je Aufgabe in USD: 0.007378. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 16.257. Standard error: 3.484. Cost per task in USD: 0.007378. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":3.6363636363636362,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":3.109,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.6363636363636362,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 18.744. Standardfehler: 1.253. Kosten je Aufgabe in USD: 0.014308. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 18.744. Standard error: 1.253. Cost per task in USD: 0.014308. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":39.473684210526315,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.79,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.473684210526315,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.644. Standardfehler: 0.88. Kosten je Aufgabe in USD: 0.00096. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.644. Standard error: 0.88. Cost per task in USD: 0.00096. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":1.3888888888888888,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.1-flash-lite-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3888888888888888,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-flash-lite-preview. Name des Quellmodells: Gemini 3.1 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 572.711. Standardfehler: 0. Kosten je Aufgabe in USD: 0.51. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-flash-lite-preview. Source model name: Gemini 3.1 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 572.711. Standard error: 0. Cost per task in USD: 0.51. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":32,"totalFamilies":71,"sourceCount":3,"coverage":0.39937641723356027,"lowerBound":14.532028676846675,"upperBound":74.59438695349064,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.3555555555555556,"lowerBound":6.282150644102304,"upperBound":70.72659508854674,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":15.269117916743074,"upperBound":72.41197505960021,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":49.38232499208109,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":32.92154999472073,"upperBound":66.25488332805406,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":8.130698462855074,"upperBound":66.46403179618841,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":18.84057971014493,"upperBound":98.84057971014494,"provisional":true,"configurationsDiffer":true},"finance":{"score":24.78953557900926,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":21.248173353436506,"upperBound":35.5338876391508,"provisional":true,"configurationsDiffer":true},"legal":{"score":24.32114333753678,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":18.240857503152583,"upperBound":43.24085750315258,"provisional":true,"configurationsDiffer":true},"health":{"score":35.77699736611062,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":23.85133157740708,"upperBound":57.184664910740416,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":11.345934650455927,"upperBound":86.34593465045593,"provisional":true,"configurationsDiffer":true},"education":{"score":47.31800766283524,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":47.31800766283524,"upperBound":47.31800766283524,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.1-pro","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":48.68421052631579,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":41.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.68421052631579,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 41,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 41.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":26.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":456.5,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":82,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 82,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 82.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":65.38461538461539,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":32.0058997050148,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.38461538461539,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 32,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 32.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":35.427718398098364,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 35,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 35.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":17.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 17,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 17.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":31.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":791.8,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":92.7536231884058,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":77.1428571428571,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.7536231884058,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 77,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 77.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":17.142857142857142,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":30.3309120258273,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.142857142857142,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 30,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 30.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":31.883333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"enigma-eval","area":"reasoning","label":{"de":"EnigmaEval","en":"EnigmaEval"},"weight":0.01020408163265306,"coverage":1,"score":50,"measurements":[{"observationId":"curated:scale-enigma-eval-2026-07-23-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/enigma_eval","benchmarkId":"enigma-eval","benchmarkVersion":"open-source release 2026-07-23","comparisonCohort":"scale-enigma-eval-2026-07-23","retrievedAt":"2026-08-26","value":36.78,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":50,"participants":5,"methodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-preview-high. Evaluator: Scale AI. Datensatztyp: public. Stand der Rangliste: 2026-07-23. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 36,78 % ± 2,71. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-76e5ed81c010. Prüfsumme der Quelldaten: 76e5ed81c01072454e0a2a0f3d05843c5c948e07565a31b7d9b7bbfd493ca0df. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-preview-high. Evaluator: Scale AI. Dataset type: public. Leaderboard date: 2026-07-23. Attempts: pass@1. Reasoning effort: high. Score display: 36.78% ± 2.71. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-76e5ed81c010. Source payload hash: 76e5ed81c01072454e0a2a0f3d05843c5c948e07565a31b7d9b7bbfd493ca0df. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.","en":"Complex multimodal puzzles with exact answer matching and one attempt."},"scoreRunConfiguration":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Versuche: pass@1. Reasoning-Stufe: high","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-enigma-eval-2026-07-23","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":100,"measurements":[{"observationId":"vals-gpqa:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":95.454,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535. Latenz in Sekunden: 65.76. Standardfehler: 1.046. Kosten je Aufgabe in USD: 0.081442. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535. Latency in seconds: 65.76. Standard error: 1.046. Cost per task in USD: 0.081442. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":87.5,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gemini-3.1-pro","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":76.2,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":87.5,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-5beb5d872d82. Prüfsumme der Quelldaten: 5beb5d872d8245204ec7f426f3063f94418e0cac67065ec663ddda909522a958. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-5beb5d872d82. Source payload hash: 5beb5d872d8245204ec7f426f3063f94418e0cac67065ec663ddda909522a958. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":87.5,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gemini-3.1-pro","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":82.1,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":87.5,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-a406d7a39da1. Prüfsumme der Quelldaten: a406d7a39da1f965402194128014f1c599f333bf35d39024f7ef5f3707e0ffbf. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-a406d7a39da1. Source payload hash: a406d7a39da1f965402194128014f1c599f333bf35d39024f7ef5f3707e0ffbf. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":47.0342910101946,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 47,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 47.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"curated:hle-scale-public-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":46.44,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-preview-thinking-high. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 46,44 % ± 1,96. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-6c4a85e66962. Prüfsumme der Quelldaten: 6c4a85e66962b6e2ca4505e87a8133d9a6bc75747627dd21bb2a1a3a018d9801. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-preview-thinking-high. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 46.44% ± 1.96. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-6c4a85e66962. Source payload hash: 6c4a85e66962b6e2ca4505e87a8133d9a6bc75747627dd21bb2a1a3a018d9801. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.1-pro-preview","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1446.7021583816515,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":22.22222222222222,"participants":37,"methodology":{"de":"Arena-Bewertung aus 24.269 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-preview. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 24,269 published pairwise votes.. Source model ID: lmarena/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-preview. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 24.269 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 24,269 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 24.269 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 24,269 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"lmarena:search:overall:gemini-3.1-pro-grounding","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1208.8056697991265,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":30.76923076923077,"participants":14,"methodology":{"de":"Arena-Bewertung aus 151.436 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.1-pro-grounding. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-grounding. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 151,436 published pairwise votes.. Source model ID: lmarena/gemini-3.1-pro-grounding. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-grounding. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 151.436 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 151,436 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 151.436 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 151,436 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":75,"measurements":[{"observationId":"lmarena:overall:gemini-3.1-pro-preview","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1487.081240890409,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":75,"participants":37,"methodology":{"de":"Arena-Bewertung aus 126.781 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-preview. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 126,781 published pairwise votes.. Source model ID: lmarena/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-preview. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 126.781 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 126,781 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 126.781 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 126,781 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":56.666666666666664,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.1-pro-preview","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1279.7787313231265,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":56.666666666666664,"participants":31,"methodology":{"de":"Arena-Bewertung aus 46.320 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: gemini-3.1-pro-preview. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 46,320 published pairwise votes.. Source model ID: lmarena/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: gemini-3.1-pro-preview. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 46.320 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 46,320 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 46.320 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 46,320 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":81.4969604863222,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":82.4277456647399,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.14285714285714,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 82,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 82.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.85106382978724,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535. Latenz in Sekunden: 76.99. Standardfehler: 0.775. Kosten je Aufgabe in USD: 0.064084. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535. Latency in seconds: 76.99. Standard error: 0.775. Cost per task in USD: 0.064084. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-multichallenge-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":71.37,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 71,37 % ± 1,74. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-239beacde521. Prüfsumme der Quelldaten: 239beacde52165a00aec3d4ba19b13edf9341eb32276703fb9cb2b9f696e64c2. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 71.37% ± 1.74. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-239beacde521. Source payload hash: 239beacde52165a00aec3d4ba19b13edf9341eb32276703fb9cb2b9f696e64c2. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"curated:scale-multinrc-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":64.74,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 64,74 % ± 2,88. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-fb4d7f469e00. Prüfsumme der Quelldaten: fb4d7f469e00c2d15559045663bf7adecaccb5dbad2699c5fc97195c09178ba0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 64.74% ± 2.88. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-fb4d7f469e00. Source payload hash: fb4d7f469e00c2d15559045663bf7adecaccb5dbad2699c5fc97195c09178ba0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"curated:scale-prbench-finance-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":41.87,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 41,87 % ± 1,22. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-1f6924438703. Prüfsumme der Quelldaten: 1f692443870315023a8bd64921cb36f5ef993171cd6be4caf509946a6166523a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 41.87% ± 1.22. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-1f6924438703. Source payload hash: 1f692443870315023a8bd64921cb36f5ef993171cd6be4caf509946a6166523a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":40,"measurements":[{"observationId":"curated:scale-prbench-legal-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":44.02,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":40,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 44,02 % ± 1,12. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-e587bb869a61. Prüfsumme der Quelldaten: e587bb869a6115290cd41ed84f0ae115b1f455736e5f227f88eb38e2caf11a2d. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 44.02% ± 1.12. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-e587bb869a61. Source payload hash: e587bb869a6115290cd41ed84f0ae115b1f455736e5f227f88eb38e2caf11a2d. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":63.559322033898304,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":78.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.559322033898304,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 312.261. Standardfehler: 1.83. Kosten je Aufgabe in USD: 0.783381. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 312.261. Standard error: 1.83. Cost per task in USD: 0.783381. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":57.1571135595526,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:bbd93ebe-80da-4594-bb19-61e69d0331df","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.04040404040404,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Modellstand: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 4,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bbd93ebe-80da-4594-bb19-61e69d0331df/gemini-3-1-pro-preview/gemini-3-1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Model snapshot: Gemini 3.1 Pro Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 4.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.1 Pro Preview (Release gemini-3-1-pro-preview, Zeile gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.1 Pro Preview (release gemini-3-1-pro-preview, row gemini-3-1-pro-preview, UUID bbd93ebe-80da-4594-bb19-61e69d0331df, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":67.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 542.887. Standardfehler: 4.996. Kosten je Aufgabe in USD: 0.516084. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 542.887. Standard error: 4.996. Cost per task in USD: 0.516084. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":70.787,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.142857142857146,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 410.249. Standardfehler: 1.124. Kosten je Aufgabe in USD: 0.582677. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 410.249. Standard error: 1.124. Cost per task in USD: 0.582677. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":2.525,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.864864864864865,"participants":38,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1124.176. Standardfehler: 0.505. Kosten je Aufgabe in USD: 4.111957. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1124.176. Standard error: 0.505. Cost per task in USD: 4.111957. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":14.516129032258064,"measurements":[{"observationId":"vals-terminal-bench-science:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":1.429,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.516129032258064,"participants":32,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2638.065. Standardfehler: 1.429. Kosten je Aufgabe in USD: 5.907095. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2638.065. Standard error: 1.429. Cost per task in USD: 5.907095. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"curated:scale-tutorbench-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":52.99,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":44.44444444444444,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 52,99 % ± 1,06. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-d1ac0053ada2. Prüfsumme der Quelldaten: d1ac0053ada2090fbdb6d6cbeb459927762a064dbf544d735e1b0bdaff785b6a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 52.99% ± 1.06. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-d1ac0053ada2. Source payload hash: d1ac0053ada2090fbdb6d6cbeb459927762a064dbf544d735e1b0bdaff785b6a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":5.2631578947368425,"measurements":[{"observationId":"vals-biomysterybench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":60,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.2631578947368425,"participants":20,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 251.867. Standardfehler: 2.94. Kosten je Aufgabe in USD: 0.713378. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 251.867. Standard error: 2.94. Cost per task in USD: 0.713378. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":33.87096774193548,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":17.313,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.87096774193548,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 720.574. Standardfehler: 3.933. Kosten je Aufgabe in USD: 1.614622. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 720.574. Standard error: 3.933. Cost per task in USD: 1.614622. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":68.91891891891892,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":64.491,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.91891891891892,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 25.388. Standardfehler: 0.942. Kosten je Aufgabe in USD: 0.181957. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 25.388. Standard error: 0.942. Cost per task in USD: 0.181957. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":32.20338983050848,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":52.616,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.20338983050848,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 708.582. Standardfehler: 2.974. Kosten je Aufgabe in USD: 3.769246. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 708.582. Standard error: 2.974. Cost per task in USD: 3.769246. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":42.982,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 208.258. Standardfehler: 1.206. Kosten je Aufgabe in USD: 1.591906. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 208.258. Standard error: 1.206. Cost per task in USD: 1.591906. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":39.39393939393939,"measurements":[{"observationId":"vals-ioi:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":51.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.39393939393939,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 722.637. Standardfehler: 2.589. Kosten je Aufgabe in USD: 1.25149. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 722.637. Standard error: 2.589. Cost per task in USD: 1.25149. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":93.50649350649351,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":88.485,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.50649350649351,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 88.891. Standardfehler: 0.931. Kosten je Aufgabe in USD: 0.101147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 88.891. Standard error: 0.931. Cost per task in USD: 0.101147. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 436.69. Standardfehler: 0. Kosten je Aufgabe in USD: 1.149757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 436.69. Standard error: 0. Cost per task in USD: 1.149757. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":96.15384615384616,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":87.398,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.15384615384616,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 10.062. Standardfehler: 0.329. Kosten je Aufgabe in USD: 0.006771. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 10.062. Standard error: 0.329. Cost per task in USD: 0.006771. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":30,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":20.673,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 242.05. Standardfehler: 2.815. Kosten je Aufgabe in USD: 1.100145. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 242.05. Standard error: 2.815. Cost per task in USD: 1.100145. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":98.50746268656717,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":59.062,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.50746268656717,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 38.523. Standardfehler: 1.996. Kosten je Aufgabe in USD: 0.024714. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 38.523. Standard error: 1.996. Cost per task in USD: 0.024714. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":97.43589743589743,"measurements":[{"observationId":"vals-medqa:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.367,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.43589743589743,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 38.67. Standardfehler: 0.171. Kosten je Aufgabe in USD: 0.017689. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 38.67. Standard error: 0.171. Cost per task in USD: 0.017689. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":26.470588235294116,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":76.114,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.470588235294116,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 69.139. Standardfehler: 1.915. Kosten je Aufgabe in USD: 0.097954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 69.139. Standard error: 1.915. Cost per task in USD: 0.097954. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":96.1038961038961,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":90.987,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.1038961038961,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535. Latenz in Sekunden: 23.912. Standardfehler: 0.284. Kosten je Aufgabe in USD: 0.027545. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535. Latency in seconds: 23.912. Standard error: 0.284. Cost per task in USD: 0.027545. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":90.38461538461539,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":69.396,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.38461538461539,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 23.616. Standardfehler: 0.908. Kosten je Aufgabe in USD: 0.023542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 23.616. Standard error: 0.908. Cost per task in USD: 0.023542. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":15,"measurements":[{"observationId":"vals-mysterymechanism:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":20.721,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15,"participants":21,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 360.456. Standardfehler: 2.726. Kosten je Aufgabe in USD: 1.065147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 360.456. Standard error: 2.726. Cost per task in USD: 1.065147. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 755.928. Standardfehler: 0. Kosten je Aufgabe in USD: 2.629095. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 755.928. Standard error: 0. Cost per task in USD: 2.629095. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":32.05128205128205,"measurements":[{"observationId":"vals-proof-bench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":26,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.05128205128205,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 889.059. Standardfehler: 4.408. Kosten je Aufgabe in USD: 0.784253. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 889.059. Standard error: 4.408. Cost per task in USD: 0.784253. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":63.793103448275865,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":48.677,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.793103448275865,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 61.174. Standardfehler: 3.293. Kosten je Aufgabe in USD: 0.068908. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 61.174. Standard error: 3.293. Cost per task in USD: 0.068908. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":10.526315789473685,"measurements":[{"observationId":"vals-snap:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":53.789,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.526315789473685,"participants":39,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 185.553. Standardfehler: 1.297. Kosten je Aufgabe in USD: 0.376768. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 185.553. Standard error: 1.297. Cost per task in USD: 0.376768. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":4.166666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.166666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3803.735. Standardfehler: 0. Kosten je Aufgabe in USD: 11.336961. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3803.735. Standard error: 0. Cost per task in USD: 11.336961. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":20,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":9.295,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 100.956. Standardfehler: 2.092. Kosten je Aufgabe in USD: 0.301555. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 100.956. Standard error: 2.092. Cost per task in USD: 0.301555. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":51.31578947368421,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.882,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.31578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535. Latenz in Sekunden: 41.931. Standardfehler: 0.863. Kosten je Aufgabe in USD: 0.03149. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535. Latency in seconds: 41.931. Standard error: 0.863. Cost per task in USD: 0.03149. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65535","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65535"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"vals-vcb-1-100:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":6.694,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":22,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 688.056. Standardfehler: 2.398. Kosten je Aufgabe in USD: 3.358183. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 688.056. Standard error: 2.398. Cost per task in USD: 3.358183. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.1-pro-preview","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":32.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.1-pro-preview. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1211.906. Standardfehler: 4.342. Kosten je Aufgabe in USD: 3.825202. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.1-pro-preview. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1211.906. Standard error: 4.342. Cost per task in USD: 3.825202. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"curated:scale-visual-toolbench-gemini-3.1-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":28.97,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":88.88888888888889,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/gemini-3.1-pro. Name des Quellmodells: Gemini 3.1 Pro Preview. Anbieter laut Quelle: Google. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 28,97 % ± 0,91. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-8a503e1f0fa4. Prüfsumme der Quelldaten: 8a503e1f0fa4886cc09972744c641863ceb6bc92f0ec72627641ee961d9565cc. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/gemini-3.1-pro. Source model name: Gemini 3.1 Pro Preview. Provider reported by source: Google. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 28.97% ± 0.91. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-8a503e1f0fa4. Source payload hash: 8a503e1f0fa4886cc09972744c641863ceb6bc92f0ec72627641ee961d9565cc. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":55.334216055150854,"coveredFamilies":54,"totalFamilies":71,"sourceCount":5,"coverage":0.7620748299319727,"lowerBound":42.16881328964812,"upperBound":65.96133029645085,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":38.11239902321356,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.6000000000000002,"lowerBound":22.867439413928142,"upperBound":62.867439413928125,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":79.50010078613184,"coveredFamilies":6,"totalFamilies":7,"sourceCount":3,"coverage":0.7857142857142857,"lowerBound":62.46436490338929,"upperBound":83.89293633196073,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.410256410256411,"upperBound":86.41025641025641,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":95.00316756414318,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":63.33544504276212,"upperBound":96.66877837609545,"provisional":true,"configurationsDiffer":true},"agentic":{"score":53.21548477196743,"coveredFamilies":8,"totalFamilies":12,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":35.47698984797829,"upperBound":68.81032318131162,"provisional":true,"configurationsDiffer":true},"general":{"score":38.43668954996187,"coveredFamilies":5,"totalFamilies":5,"sourceCount":3,"coverage":1,"lowerBound":38.43668954996187,"upperBound":38.43668954996187,"provisional":true,"configurationsDiffer":true},"finance":{"score":46.117530515389575,"coveredFamilies":7,"totalFamilies":7,"sourceCount":2,"coverage":1,"lowerBound":46.117530515389575,"upperBound":46.117530515389575,"provisional":true,"configurationsDiffer":true},"legal":{"score":44.20239596469104,"coveredFamilies":4,"totalFamilies":4,"sourceCount":2,"coverage":1,"lowerBound":44.20239596469104,"upperBound":44.20239596469104,"provisional":true,"configurationsDiffer":true},"health":{"score":74.13798278591956,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":74.13798278591956,"upperBound":74.13798278591956,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":75.68417201395924,"coveredFamilies":3,"totalFamilies":4,"sourceCount":4,"coverage":0.75,"lowerBound":56.763129010469434,"upperBound":81.76312901046944,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":30.769230769230766,"upperBound":30.769230769230766,"provisional":true,"configurationsDiffer":true},"productivity":{"score":55.26315789473684,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":55.26315789473684,"upperBound":55.26315789473684,"provisional":true,"configurationsDiffer":true},"education":{"score":54.11877394636016,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":54.11877394636016,"upperBound":54.11877394636016,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.5-flash","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":13.636363636363637,"measurements":[{"observationId":"research:gemini-3.5-flash:22","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.8888888888888888,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":13.636363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_low. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: low. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: ig7gQ23WRms9SQLLNPtbTi. Startzeit: 2026-08-06T22:34:49.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.8888888888888888 und stderr=0.047377936967913425. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: low.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_low; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: low. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: ig7gQ23WRms9SQLLNPtbTi. Started at 2026-08-06T22:34:49.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.8888888888888888 and stderr=0.047377936967913425. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: low."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: low. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: low. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"low","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":60.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":45,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 45,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 45.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":874.32,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":73.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 73,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 73.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":47.0501474926254,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 47,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 47.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":42.07973565652745,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 42,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 42.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":75.60975609756098,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":19.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 19,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 19.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":57.31707317073171,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1202.58,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.31707317073171,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":91.30434782608695,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":76.32653061224491,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.30434782608695,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 76,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 76.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":57.142857142857146,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":40.3483992467043,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.142857142857146,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 40,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 40.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":21.183333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"enigma-eval","area":"reasoning","label":{"de":"EnigmaEval","en":"EnigmaEval"},"weight":0.01020408163265306,"coverage":1,"score":25,"measurements":[{"observationId":"curated:scale-enigma-eval-2026-07-23-gemini-3.5-flash","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/enigma_eval","benchmarkId":"enigma-eval","benchmarkVersion":"open-source release 2026-07-23","comparisonCohort":"scale-enigma-eval-2026-07-23","retrievedAt":"2026-08-26","value":25.41,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":25,"participants":5,"methodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Kennung des Quellmodells: curated/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-high. Evaluator: Scale AI. Datensatztyp: public. Stand der Rangliste: 2026-07-23. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 25,41 % ± 2,45. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-42f1970146b9. Prüfsumme der Quelldaten: 42f1970146b9e570d91fd2312f41b9bc5b47ac0633cc41fcf2201ead2e816f1f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Source model ID: curated/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-high. Evaluator: Scale AI. Dataset type: public. Leaderboard date: 2026-07-23. Attempts: pass@1. Reasoning effort: high. Score display: 25.41% ± 2.45. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-42f1970146b9. Source payload hash: 42f1970146b9e570d91fd2312f41b9bc5b47ac0633cc41fcf2201ead2e816f1f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.","en":"Complex multimodal puzzles with exact answer matching and one attempt."},"scoreRunConfiguration":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Versuche: pass@1. Reasoning-Stufe: high","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-enigma-eval-2026-07-23","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":54.76190476190476,"measurements":[{"observationId":"research:gemini-3.5-flash:18","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.45,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":54.76190476190476,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_high. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: XUDdWKUZBhdWPPQBRgx8Pw. Startzeit: 2026-10-13T13:33:53.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.45 und stderr=0.05. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_high; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: XUDdWKUZBhdWPPQBRgx8Pw. Started at 2026-10-13T13:33:53.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.45 and stderr=0.05. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":69.44444444444444,"measurements":[{"observationId":"research:gemini-3.5-flash:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.662,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":69.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_high. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: GJ6YsSuzyRD9YXe6SBwvsu. Startzeit: 2026-08-27T19:30:35.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.662 und stderr=0.014965960710224383. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_high; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: GJ6YsSuzyRD9YXe6SBwvsu. Started at 2026-08-27T19:30:35.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.662 and stderr=0.014965960710224383. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":30.303030303030305,"measurements":[{"observationId":"research:gemini-3.5-flash:15","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.2682926829268293,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":33.333333333333336,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_high. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: cXn5zzns6VFwbTgotDwZjV. Startzeit: 2026-06-10T17:32:38.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.2682926829268293 und stderr=0.07005564203095158. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_high; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: cXn5zzns6VFwbTgotDwZjV. Started at 2026-06-10T17:32:38.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.2682926829268293 and stderr=0.07005564203095158. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gemini-3.5-flash:14","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.6280701754385964,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":27.272727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_high. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: 72DQVwbYiQW5pjhshjBaGU. Startzeit: 2026-06-10T17:32:38.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.6280701754385964 und stderr=0.028679753751074618. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_high; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: 72DQVwbYiQW5pjhshjBaGU. Started at 2026-06-10T17:32:38.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.6280701754385964 and stderr=0.028679753751074618. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":48.26127819548873,"measurements":[{"observationId":"research:gemini-3.5-flash:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.8888888888888888,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":14.285714285714286,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.5-flash_low. Epoch-Modellbezeichnung: Gemini 3.5 Flash. Anzeigenamenfeld: Gemini 3.5 Flash. Im Alias kodierte Reasoning-Stufe: low. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: eiTYduCDNq3HgwTg4ZhEw6. Startzeit: 2026-08-06T22:30:55.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.8888888888888888 und stderr=0.02239078763821682. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: low.","en":"Independent evaluation. Epoch run alias: gemini-3.5-flash_low; Epoch model label: Gemini 3.5 Flash; Display label field: Gemini 3.5 Flash; Reasoning setting encoded in the alias: low. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: eiTYduCDNq3HgwTg4ZhEw6. Started at 2026-08-06T22:30:55.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.8888888888888888 and stderr=0.02239078763821682. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: low."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: low. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: low. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"low","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":92.676,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.23684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 27.655. Standardfehler: 1.462. Kosten je Aufgabe in USD: 0.063842. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 27.655. Standard error: 1.462. Cost per task in USD: 0.063842. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gemini-3.5-flash","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":79.2,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-04b16718a8e7. Prüfsumme der Quelldaten: 04b16718a8e7a8a04fb0ad6ecc2bb7b0afcf74d448c4c1076907ec22e5cb331c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-04b16718a8e7. Source payload hash: 04b16718a8e7a8a04fb0ad6ecc2bb7b0afcf74d448c4c1076907ec22e5cb331c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":68.75,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gemini-3.5-flash","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":80.8,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":68.75,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-8023c329706f. Prüfsumme der Quelldaten: 8023c329706fe5ccb7162223319f0987121f8e12c42400e3949a4c00c435852e. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-8023c329706f. Source payload hash: 8023c329706fe5ccb7162223319f0987121f8e12c42400e3949a4c00c435852e. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":75,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":42.678405931418,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 42,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 42.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":27.77777777777778,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.5-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1499.4257204597016,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":27.77777777777778,"participants":37,"methodology":{"de":"Arena-Bewertung aus 9.415 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-high. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 9,415 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-high. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 9.415 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 9,415 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 9.415 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 9,415 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":61.111111111111114,"measurements":[{"observationId":"lmarena:overall:gemini-3.5-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1477.3438128702871,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":61.111111111111114,"participants":37,"methodology":{"de":"Arena-Bewertung aus 48.486 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-high. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 48,486 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-high. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 48.486 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 48,486 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 48.486 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 48,486 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.5-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1283.6586047947885,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":66.66666666666667,"participants":31,"methodology":{"de":"Arena-Bewertung aus 13.103 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-high. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 13,103 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-high. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 13.103 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 13,103 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 13.103 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 13,103 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":87.55699088145897,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":84.2774566473988,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.07142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 84,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 84.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.266,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.04255319148936,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.226. Standardfehler: 0.774. Kosten je Aufgabe in USD: 0.025155. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.226. Standard error: 0.774. Cost per task in USD: 0.025155. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":63.559322033898304,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":78.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.559322033898304,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 254.129. Standardfehler: 1.83. Kosten je Aufgabe in USD: 0.953882. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 254.129. Standard error: 1.83. Cost per task in USD: 0.953882. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":61.135634763683555,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:0097ebf5-124f-42f6-9463-33b00e711f03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":6.56565656565657,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 6,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0097ebf5-124f-42f6-9463-33b00e711f03/gemini-3-5-flash/gemini-3-5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 6.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash (High) (Release gemini-3-5-flash, Zeile gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.5 Flash (High) (release gemini-3-5-flash, row gemini-3-5-flash, UUID 0097ebf5-124f-42f6-9463-33b00e711f03, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":67.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 408.048. Standardfehler: 1.297. Kosten je Aufgabe in USD: 1.192081. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 408.048. Standard error: 1.297. Cost per task in USD: 1.192081. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":74.157,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.64285714285714,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 373.223. Standardfehler: 1.124. Kosten je Aufgabe in USD: 0.868179. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 373.223. Standard error: 1.124. Cost per task in USD: 0.868179. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":6.061,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.91891891891892,"participants":38,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4286.824. Standardfehler: 1.515. Kosten je Aufgabe in USD: 5.981757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4286.824. Standard error: 1.515. Cost per task in USD: 5.981757. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":38.70967741935484,"measurements":[{"observationId":"vals-terminal-bench-science:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":5.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.70967741935484,"participants":32,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3654.61. Standardfehler: 2.794. Kosten je Aufgabe in USD: 3.802646. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3654.61. Standard error: 2.794. Cost per task in USD: 3.802646. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":46.774193548387096,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":26.745,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.774193548387096,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1168.454. Standardfehler: 4.104. Kosten je Aufgabe in USD: 5.725122. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1168.454. Standard error: 4.104. Cost per task in USD: 5.725122. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":70.27027027027027,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":64.686,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.27027027027027,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 8.7. Standardfehler: 0.942. Kosten je Aufgabe in USD: 0.140208. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 8.7. Standard error: 0.942. Cost per task in USD: 0.140208. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":61.016949152542374,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":63.548,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.016949152542374,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1123.555. Standardfehler: 2.752. Kosten je Aufgabe in USD: 5.173115. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1123.555. Standard error: 2.752. Cost per task in USD: 5.173115. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":57.861,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 322.279. Standardfehler: 0.231. Kosten je Aufgabe in USD: 2.505488. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 322.279. Standard error: 0.231. Cost per task in USD: 2.505488. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":87.01298701298701,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.604,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.01298701298701,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 64.371. Standardfehler: 0.95. Kosten je Aufgabe in USD: 0.085937. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 64.371. Standard error: 0.95. Cost per task in USD: 0.085937. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":46.721311475409834,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.5,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.721311475409834,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 478.414. Standardfehler: 0.833. Kosten je Aufgabe in USD: 2.116049. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 478.414. Standard error: 0.833. Cost per task in USD: 2.116049. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":55.12820512820513,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.602,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.12820512820513,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 34.576. Standardfehler: 0.849. Kosten je Aufgabe in USD: 0.007553. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 34.576. Standard error: 0.849. Cost per task in USD: 0.007553. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":30.769,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 277.753. Standardfehler: 3.208. Kosten je Aufgabe in USD: 1.305281. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 277.753. Standard error: 3.208. Cost per task in USD: 1.305281. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":94.02985074626865,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":55.825,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.02985074626865,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 25.29. Standardfehler: 2.113. Kosten je Aufgabe in USD: 0.073716. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 25.29. Standard error: 2.113. Cost per task in USD: 0.073716. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":29.41176470588235,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":76.574,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.41176470588235,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 57.075. Standardfehler: 1.923. Kosten je Aufgabe in USD: 0.166341. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 57.075. Standard error: 1.923. Cost per task in USD: 0.166341. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":89.6103896103896,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.6103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 26.262. Standardfehler: 0.306. Kosten je Aufgabe in USD: 0.02498. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 26.262. Standard error: 0.306. Cost per task in USD: 0.02498. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":71.15384615384616,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.124,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.15384615384616,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 11.428. Standardfehler: 0.91. Kosten je Aufgabe in USD: 0.017873. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 11.428. Standard error: 0.91. Cost per task in USD: 0.017873. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1104.517. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1104.517. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":37.17948717948718,"measurements":[{"observationId":"vals-proof-bench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":31,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.17948717948718,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 394.476. Standardfehler: 4.648. Kosten je Aufgabe in USD: 0.569863. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 394.476. Standard error: 4.648. Cost per task in USD: 0.569863. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":75.86206896551724,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":49.885,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.86206896551724,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 32.592. Standardfehler: 3.438. Kosten je Aufgabe in USD: 0.052847. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 32.592. Standard error: 3.438. Cost per task in USD: 0.052847. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":55.172413793103445,"measurements":[{"observationId":"vals-skillsbench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":52.737,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.172413793103445,"participants":30,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 435.011. Standardfehler: 4.475. Kosten je Aufgabe in USD: 1.253121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 435.011. Standard error: 4.475. Cost per task in USD: 1.253121. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":34.21052631578947,"measurements":[{"observationId":"vals-snap:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":59.472,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.21052631578947,"participants":39,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 338.872. Standardfehler: 1.277. Kosten je Aufgabe in USD: 1.113099. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 338.872. Standard error: 1.277. Cost per task in USD: 1.113099. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":43.63636363636363,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":21.652,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.63636363636363,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 233.001. Standardfehler: 2.963. Kosten je Aufgabe in USD: 0.859679. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 233.001. Standard error: 2.963. Cost per task in USD: 0.859679. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":67.10526315789474,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.366,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.10526315789474,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 19.694. Standardfehler: 0.846. Kosten je Aufgabe in USD: 0.038136. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 19.694. Standard error: 0.846. Cost per task in USD: 0.038136. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":47.22222222222222,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.5-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":48.683,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.22222222222222,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash. Name des Quellmodells: Gemini 3.5 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 893.074. Standardfehler: 4.727. Kosten je Aufgabe in USD: 2.540385. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash. Source model name: Gemini 3.5 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 893.074. Standard error: 4.727. Cost per task in USD: 2.540385. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":60.29625566494994,"coveredFamilies":46,"totalFamilies":71,"sourceCount":6,"coverage":0.5554421768707485,"lowerBound":33.491083503694995,"upperBound":77.94686581662015,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.4666666666666669,"lowerBound":23.036474064670113,"upperBound":76.36980739800342,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":50.49423725611146,"coveredFamilies":6,"totalFamilies":7,"sourceCount":4,"coverage":0.7857142857142857,"lowerBound":39.67404355837329,"upperBound":61.10261498694472,"provisional":true,"configurationsDiffer":true},"math":{"score":27.039627039627035,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":16.223776223776223,"upperBound":56.22377622377621,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":79.44104224592029,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":79.44104224592029,"upperBound":79.44104224592029,"provisional":true,"configurationsDiffer":true},"agentic":{"score":68.03592156027861,"coveredFamilies":9,"totalFamilies":12,"sourceCount":3,"coverage":0.75,"lowerBound":51.02694117020896,"upperBound":76.02694117020896,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":30.483091787439616,"upperBound":90.48309178743962,"provisional":true,"configurationsDiffer":true},"finance":{"score":66.64155983959732,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":57.12133700536913,"upperBound":71.40705129108342,"provisional":true,"configurationsDiffer":true},"legal":{"score":49.50539442342721,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":37.12904581757041,"upperBound":62.12904581757041,"provisional":true,"configurationsDiffer":true},"health":{"score":61.720807726075506,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":41.14720515071701,"upperBound":74.48053848405034,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":77.11182877406283,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":38.555914387031414,"upperBound":88.55591438703141,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":17.105263157894736,"upperBound":67.10526315789474,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":37.93103448275862,"upperBound":87.93103448275862,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.5-flash-lite","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":650.26,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":76,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 76,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 76.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":56.09756097560975,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":25.011270511840955,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 25,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 25.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":13.600000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 13,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 13.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":983.28,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":5.233333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":47.36842105263158,"measurements":[{"observationId":"vals-gpqa:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":83.838,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 13.848. Standardfehler: 1.847. Kosten je Aufgabe in USD: 0.01235. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 13.848. Standard error: 1.847. Cost per task in USD: 0.01235. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":18.813716404077798,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 18,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 18.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":19.444444444444443,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.5-flash-lite","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1440.4005622297457,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":19.444444444444443,"participants":37,"methodology":{"de":"Arena-Bewertung aus 242 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-lite. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 242 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-lite. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 242 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 242 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 242 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 242 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"lmarena:overall:gemini-3.5-flash-lite","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1454.971388095603,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":22.22222222222222,"participants":37,"methodology":{"de":"Arena-Bewertung aus 35.845 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-lite. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 35,845 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-lite. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 35.845 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 35,845 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 35.845 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 35,845 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":40,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.5-flash-lite","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1265.9991699575933,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":40,"participants":31,"methodology":{"de":"Arena-Bewertung aus 8.381 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: gemini-3.5-flash-lite. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 8,381 published pairwise votes.. Source model ID: lmarena/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: gemini-3.5-flash-lite. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 8.381 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 8,381 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 8.381 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 8,381 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":59.099544072948326,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":79.0173410404624,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":66.07142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 79,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 79.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.584,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.12765957446808,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.799. Standardfehler: 0.891. Kosten je Aufgabe in USD: 0.006577. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.799. Standard error: 0.891. Cost per task in USD: 0.006577. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":45.76271186440678,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.76271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 439.669. Standardfehler: 1.938. Kosten je Aufgabe in USD: 0.30898. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 439.669. Standard error: 1.938. Cost per task in USD: 0.30898. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":41.95339721254356,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":50.187,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.642857142857142,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 416.383. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.337052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 416.383. Standard error: 0.991. Cost per task in USD: 0.337052. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":4.3478260869565215,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:093b9df2-ebee-4309-943f-5a7e88360adf","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.3478260869565215,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Modellstand: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/093b9df2-ebee-4309-943f-5a7e88360adf/gemini-3-5-flash-lite/gemini-3-5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Model snapshot: Gemini 3.5 Flash-Lite. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.5 Flash-Lite (Release gemini-3-5-flash-lite, Zeile gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Gemini 3.5 Flash-Lite (release gemini-3-5-flash-lite, row gemini-3-5-flash-lite, UUID 093b9df2-ebee-4309-943f-5a7e88360adf, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":11.290322580645162,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":6.148,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.290322580645162,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1322.302. Standardfehler: 1.708. Kosten je Aufgabe in USD: 0.525186. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1322.302. Standard error: 1.708. Cost per task in USD: 0.525186. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":39.189189189189186,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.684,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.189189189189186,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3.836. Standardfehler: 0.962. Kosten je Aufgabe in USD: 0.028548. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3.836. Standard error: 0.962. Cost per task in USD: 0.028548. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-cyberbench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":59.107,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1075.887. Standardfehler: 6.17. Kosten je Aufgabe in USD: 0.425124. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1075.887. Standard error: 6.17. Cost per task in USD: 0.425124. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":22.033898305084747,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":43.231,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.033898305084747,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 493.632. Standardfehler: 3.105. Kosten je Aufgabe in USD: 1.231783. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 493.632. Standard error: 3.105. Cost per task in USD: 1.231783. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":36.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":47.442,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 142.913. Standardfehler: 0.513. Kosten je Aufgabe in USD: 0.389575. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 142.913. Standard error: 0.513. Cost per task in USD: 0.389575. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":28.571428571428573,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":79.014,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.571428571428573,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 21.084. Standardfehler: 1.098. Kosten je Aufgabe in USD: 0.019272. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 21.084. Standard error: 1.098. Cost per task in USD: 0.019272. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 232.895. Standardfehler: 0. Kosten je Aufgabe in USD: 0.331805. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 232.895. Standard error: 0. Cost per task in USD: 0.331805. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":67.3076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.073,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.3076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1.666. Standardfehler: 0.437. Kosten je Aufgabe in USD: 0.00128. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1.666. Standard error: 0.437. Cost per task in USD: 0.00128. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":15.833333333333334,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":13.942,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.833333333333334,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 59.197. Standardfehler: 2.408. Kosten je Aufgabe in USD: 0.124723. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 59.197. Standard error: 2.408. Cost per task in USD: 0.124723. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":52.23880597014925,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":43.489,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.23880597014925,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 6.019. Standardfehler: 1.951. Kosten je Aufgabe in USD: 0.008093. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 6.019. Standard error: 1.951. Cost per task in USD: 0.008093. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":13.235294117647058,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":70.886,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.235294117647058,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 18.991. Standardfehler: 2.031. Kosten je Aufgabe in USD: 0.019867. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 18.991. Standard error: 2.031. Cost per task in USD: 0.019867. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":51.94805194805195,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.836,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.94805194805195,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.823. Standardfehler: 0.344. Kosten je Aufgabe in USD: 0.004522. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.823. Standard error: 0.344. Cost per task in USD: 0.004522. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":79.8076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.68,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.8076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.879. Standardfehler: 0.914. Kosten je Aufgabe in USD: 0.004323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.879. Standard error: 0.914. Cost per task in USD: 0.004323. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 434.999. Standardfehler: 0. Kosten je Aufgabe in USD: 0.337485. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 434.999. Standard error: 0. Cost per task in USD: 0.337485. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":56.89655172413793,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":47.303,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.89655172413793,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 10.751. Standardfehler: 3.384. Kosten je Aufgabe in USD: 0.010542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 10.751. Standard error: 3.384. Cost per task in USD: 0.010542. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":7.894736842105263,"measurements":[{"observationId":"vals-snap:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":51.827,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.894736842105263,"participants":39,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 69.26. Standardfehler: 1.3. Kosten je Aufgabe in USD: 0.09565. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 69.26. Standard error: 1.3. Cost per task in USD: 0.09565. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":14.545454545454545,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":7.254,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.545454545454545,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 47.461. Standardfehler: 1.872. Kosten je Aufgabe in USD: 0.081536. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 47.461. Standard error: 1.872. Cost per task in USD: 0.081536. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":48.68421052631579,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.608,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.68421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 8.815. Standardfehler: 0.875. Kosten je Aufgabe in USD: 0.00729. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 8.815. Standard error: 0.875. Cost per task in USD: 0.00729. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":36.111111111111114,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.5-flash-lite","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":37.161,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.111111111111114,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.5-flash-lite. Name des Quellmodells: Gemini 3.5 Flash-Lite. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 501.522. Standardfehler: 4.627. Kosten je Aufgabe in USD: 0.833111. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.5-flash-lite. Source model name: Gemini 3.5 Flash-Lite. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 501.522. Standard error: 4.627. Cost per task in USD: 0.833111. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":34,"totalFamilies":71,"sourceCount":3,"coverage":0.4831065759637189,"lowerBound":17.853672788557073,"upperBound":69.54301519218518,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.44444444444444453,"lowerBound":11.528753538447157,"upperBound":67.0843090940027,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":20.108197322574732,"upperBound":77.25105446543188,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":58.29109914475769,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":38.8607327631718,"upperBound":72.19406609650514,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":10.873983739837398,"upperBound":85.8739837398374,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.444444444444445,"upperBound":84.44444444444444,"provisional":true,"configurationsDiffer":true},"finance":{"score":40.15451859006721,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":34.41815879148618,"upperBound":48.70387307720047,"provisional":true,"configurationsDiffer":true},"legal":{"score":31.265587781981225,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":23.44919083648592,"upperBound":48.44919083648592,"provisional":true,"configurationsDiffer":true},"health":{"score":32.73705004389816,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":21.824700029265436,"upperBound":55.15803336259877,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":27.272727272727273,"upperBound":27.272727272727273,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":49.54977203647416,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":24.77488601823708,"upperBound":74.77488601823708,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":3.947368421052631,"upperBound":53.94736842105263,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":28.448275862068964,"upperBound":78.44827586206897,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.6-flash","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"research:gemini-3.6-flash:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9416666666666667,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":27.272727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: ajQFZGMjvyMTH2ULDZhwtz. Startzeit: 2026-08-02T00:21:27.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9416666666666667 und stderr=0.030926354359015452. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: ajQFZGMjvyMTH2ULDZhwtz. Started at 2026-08-02T00:21:27.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9416666666666667 and stderr=0.030926354359015452. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":57.31707317073171,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":953.92,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.31707317073171,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":80,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 80,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 80.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":53.01669491509355,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 53,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 53.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":17.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 17,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 17.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1284.34,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":22.133333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":38.095238095238095,"measurements":[{"observationId":"research:gemini-3.6-flash:14","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.4,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":38.095238095238095,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: ZDeGG5QYkdgVqzGPmmnbfx. Startzeit: 2026-10-13T13:14:52.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.4 und stderr=0.0492365963917331. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: ZDeGG5QYkdgVqzGPmmnbfx. Started at 2026-10-13T13:14:52.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.4 and stderr=0.0492365963917331. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":69.44444444444444,"measurements":[{"observationId":"research:gemini-3.6-flash:16","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.662,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":69.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: F83pa7Gfsb4UNVttTnS5Hk. Startzeit: 2026-08-27T19:30:09.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.662 und stderr=0.014965960710224383. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: F83pa7Gfsb4UNVttTnS5Hk. Started at 2026-08-27T19:30:09.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.662 and stderr=0.014965960710224383. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":18.813131313131315,"measurements":[{"observationId":"research:gemini-3.6-flash:11","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.21951219512195122,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":19.444444444444443,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: JCMjXU6FRiiLV3xvs9R2Xv. Startzeit: 2026-08-02T00:36:45.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.21951219512195122 und stderr=0.06544589202438408. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: JCMjXU6FRiiLV3xvs9R2Xv. Started at 2026-08-02T00:36:45.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.21951219512195122 and stderr=0.06544589202438408. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gemini-3.6-flash:10","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.5894736842105263,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":18.181818181818183,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: nb9LnDeeQyvRCqsbsTANGD. Startzeit: 2026-08-02T00:36:45.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.5894736842105263 und stderr=0.02919063494391404. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: nb9LnDeeQyvRCqsbsTANGD. Started at 2026-08-02T00:36:45.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.5894736842105263 and stderr=0.02919063494391404. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":81.10902255639098,"measurements":[{"observationId":"research:gemini-3.6-flash:12","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9412878787878788,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":71.42857142857143,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.6-flash_high. Epoch-Modellbezeichnung: Gemini 3.6 Flash. Anzeigenamenfeld: Gemini 3.6 Flash. Im Alias kodierte Reasoning-Stufe: high. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: 86SY5JbPB49xXhHFQW6tHu. Startzeit: 2026-08-02T00:21:27.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9412878787878788 und stderr=0.014028967942064968. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.6-flash_high; Epoch model label: Gemini 3.6 Flash; Display label field: Gemini 3.6 Flash; Reasoning setting encoded in the alias: high. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: 86SY5JbPB49xXhHFQW6tHu. Started at 2026-08-02T00:21:27.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9412878787878788 and stderr=0.014028967942064968. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.78947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 18.064. Standardfehler: 1.329. Kosten je Aufgabe in USD: 0.03189. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 18.064. Standard error: 1.329. Cost per task in USD: 0.03189. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":40.824837812789596,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 40,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 40.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":36.111111111111114,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.6-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1538.0441316875786,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":36.111111111111114,"participants":37,"methodology":{"de":"Arena-Bewertung aus 8.768 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.6-flash-high. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.6-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 8,768 published pairwise votes.. Source model ID: lmarena/gemini-3.6-flash-high. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: gemini-3.6-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 8.768 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 8,768 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 8.768 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 8,768 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"lmarena:overall:gemini-3.6-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1482.9109473613437,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":66.66666666666667,"participants":37,"methodology":{"de":"Arena-Bewertung aus 36.107 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.6-flash-high. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.6-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 36,107 published pairwise votes.. Source model ID: lmarena/gemini-3.6-flash-high. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: gemini-3.6-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 36.107 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 36,107 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 36.107 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 36,107 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":63.333333333333336,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.6-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1280.7135090511688,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":63.333333333333336,"participants":31,"methodology":{"de":"Arena-Bewertung aus 8.127 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.6-flash-high. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.6-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 8,127 published pairwise votes.. Source model ID: lmarena/gemini-3.6-flash-high. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: gemini-3.6-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 8.127 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 8,127 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 8.127 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 8,127 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":87.36702127659575,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":83.2369942196532,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.5,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 83,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 83.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.382,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.23404255319149,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 11.313. Standardfehler: 0.77. Kosten je Aufgabe in USD: 0.0186. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 11.313. Standard error: 0.77. Cost per task in USD: 0.0186. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":66.94915254237289,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":79.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.94915254237289,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 788.09. Standardfehler: 1.804. Kosten je Aufgabe in USD: 1.194601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 788.09. Standard error: 1.804. Cost per task in USD: 1.194601. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":64.52526132404181,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:78332c88-fe60-42c8-af9d-3617e44cf1f5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":7.07070707070707,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 7,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/78332c88-fe60-42c8-af9d-3617e44cf1f5/gemini-3-6-flash/gemini-3-6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.6 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 7.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.6 Flash (High) (Release gemini-3-6-flash, Zeile gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.6 Flash (High) (release gemini-3-6-flash, row gemini-3-6-flash, UUID 78332c88-fe60-42c8-af9d-3617e44cf1f5, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":73.783,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.85714285714286,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 552.153. Standardfehler: 1.633. Kosten je Aufgabe in USD: 1.174404. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 552.153. Standard error: 1.633. Cost per task in USD: 1.174404. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":27.419354838709676,"measurements":[{"observationId":"vals-terminal-bench-science:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":4.286,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.419354838709676,"participants":32,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 6744.655. Standardfehler: 2.438. Kosten je Aufgabe in USD: 4.390771. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 6744.655. Standard error: 2.438. Cost per task in USD: 4.390771. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"vals-biomysterybench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":58.519,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":20,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 761.958. Standardfehler: 1.614. Kosten je Aufgabe in USD: 1.252177. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 761.958. Standard error: 1.614. Cost per task in USD: 1.252177. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":53.225806451612904,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":30.928,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.225806451612904,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 5404.829. Standardfehler: 4.068. Kosten je Aufgabe in USD: 8.930416. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 5404.829. Standard error: 4.068. Cost per task in USD: 8.930416. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":60.810810810810814,"measurements":[{"observationId":"vals-corp-fin-v2:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":63.326,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.810810810810814,"participants":75,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 6.329. Standardfehler: 0.949. Kosten je Aufgabe in USD: 0.136842. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 6.329. Standard error: 0.949. Cost per task in USD: 0.136842. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-cyberbench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":45.357,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 660.036. Standardfehler: 3.745. Kosten je Aufgabe in USD: 1.306182. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 660.036. Standard error: 3.745. Cost per task in USD: 1.306182. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":66.10169491525424,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":65.411,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.10169491525424,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 726.48. Standardfehler: 2.646. Kosten je Aufgabe in USD: 3.893069. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 726.48. Standard error: 2.646. Cost per task in USD: 3.893069. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":80,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":56.296,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 224.899. Standardfehler: 0.183. Kosten je Aufgabe in USD: 1.402901. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 224.899. Standard error: 0.183. Cost per task in USD: 1.402901. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-ioi:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":35.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2903.307. Standardfehler: 5.695. Kosten je Aufgabe in USD: 8.66611. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2903.307. Standard error: 5.695. Cost per task in USD: 8.66611. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":88.08,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 31.883. Standardfehler: 0.939. Kosten je Aufgabe in USD: 0.056383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 31.883. Standard error: 0.939. Cost per task in USD: 0.056383. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":52.459016393442624,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":3.333,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.459016393442624,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 984.49. Standardfehler: 1.174. Kosten je Aufgabe in USD: 1.895602. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 984.49. Standard error: 1.174. Cost per task in USD: 1.895602. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":88.46153846153847,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.705,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.46153846153847,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2.962. Standardfehler: 0.414. Kosten je Aufgabe in USD: 0.004766. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2.962. Standard error: 0.414. Cost per task in USD: 0.004766. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":25,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 168.482. Standardfehler: 3.01. Kosten je Aufgabe in USD: 0.466741. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 168.482. Standard error: 3.01. Cost per task in USD: 0.466741. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":86.56716417910448,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":53.153,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.56716417910448,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.685. Standardfehler: 2.157. Kosten je Aufgabe in USD: 0.044216. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 16.685. Standard error: 2.157. Cost per task in USD: 0.044216. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":38.23529411764706,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":79.661,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.23529411764706,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 33.552. Standardfehler: 1.861. Kosten je Aufgabe in USD: 0.073178. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 33.552. Standard error: 1.861. Cost per task in USD: 0.073178. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.276,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.298. Standardfehler: 0.305. Kosten je Aufgabe in USD: 0.013327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.298. Standard error: 0.305. Cost per task in USD: 0.013327. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":65.38461538461539,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":67.608,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.38461538461539,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 9.239. Standardfehler: 0.915. Kosten je Aufgabe in USD: 0.019658. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 9.239. Standard error: 0.915. Cost per task in USD: 0.019658. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2852.749. Standardfehler: 0. Kosten je Aufgabe in USD: 5.923527. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2852.749. Standard error: 0. Cost per task in USD: 5.923527. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":65.51724137931035,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":48.686,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.51724137931035,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 21.96. Standardfehler: 3.383. Kosten je Aufgabe in USD: 0.045301. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 21.96. Standard error: 3.383. Cost per task in USD: 0.045301. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":15.789473684210526,"measurements":[{"observationId":"vals-snap:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":56.563,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.789473684210526,"participants":39,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 121.522. Standardfehler: 1.289. Kosten je Aufgabe in USD: 0.385496. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 121.522. Standard error: 1.289. Cost per task in USD: 0.385496. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":18.038,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 105.32. Standardfehler: 2.766. Kosten je Aufgabe in USD: 0.356293. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 105.32. Standard error: 2.766. Cost per task in USD: 0.356293. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":76.3157894736842,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.3157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.335. Standardfehler: 0.847. Kosten je Aufgabe in USD: 0.020385. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.335. Standard error: 0.847. Cost per task in USD: 0.020385. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.6-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":64.005,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.55555555555556,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.6-flash. Name des Quellmodells: Gemini 3.6 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1523.201. Standardfehler: 4.29. Kosten je Aufgabe in USD: 3.041993. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.6-flash. Source model name: Gemini 3.6 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1523.201. Standard error: 4.29. Cost per task in USD: 3.041993. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":52.53743811102506,"coveredFamilies":40,"totalFamilies":71,"sourceCount":4,"coverage":0.5698412698412699,"lowerBound":29.938000447393648,"upperBound":72.95387346326666,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":46.902177862158624,"coveredFamilies":9,"totalFamilies":15,"sourceCount":3,"coverage":0.5111111111111112,"lowerBound":23.972224240658857,"upperBound":72.86111312954773,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":64.29471374914252,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":41.33231598159161,"upperBound":77.04660169587734,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":9.217171717171718,"upperBound":69.21717171717172,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":78.9553490773003,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":78.9553490773003,"upperBound":78.9553490773003,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":15.650406504065042,"upperBound":90.65040650406505,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":13.333333333333336,"upperBound":73.33333333333334,"provisional":true,"configurationsDiffer":true},"finance":{"score":64.16275782466684,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":54.996649564000144,"upperBound":69.28236384971443,"provisional":true,"configurationsDiffer":true},"legal":{"score":58.08462939610481,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":43.563472047078605,"upperBound":68.5634720470786,"provisional":true,"configurationsDiffer":true},"health":{"score":62.40122914837577,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":41.60081943225052,"upperBound":74.93415276558386,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":18.181818181818183,"upperBound":18.181818181818183,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":75.35017730496455,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":37.67508865248227,"upperBound":87.67508865248227,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":7.894736842105262,"upperBound":57.89473684210526,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":32.758620689655174,"upperBound":82.75862068965517,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.7-flash","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":38.63636363636363,"measurements":[{"observationId":"research:gemini-3.7-flash:25","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9722222222222222,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":38.63636363636363,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: kQjrYCaa7yYyMXofeR23Dd. Startzeit: 2026-08-14T18:29:12.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9722222222222222 und stderr=0.018586784946171872. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: kQjrYCaa7yYyMXofeR23Dd. Started at 2026-08-14T18:29:12.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9722222222222222 and stderr=0.018586784946171872. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":60,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 60,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 60.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1113.53,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":81.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 81,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 81.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":62.02749292053469,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 62,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 62.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":23.599999999999998,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 23,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 23.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1391.99,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":86.58536585365853,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":26.483333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":90.47619047619048,"measurements":[{"observationId":"research:gemini-3.7-flash:23","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.66,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.47619047619048,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: Xu34dKTAdpBWL6advf6DMg. Startzeit: 2026-10-13T13:00:23.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.66 und stderr=0.04760952285695234. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: Xu34dKTAdpBWL6advf6DMg. Started at 2026-10-13T13:00:23.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.66 and stderr=0.04760952285695234. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"research:gemini-3.7-flash:24","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.692,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":77.77777777777777,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: 84ARuGyhQsjgh8umRmBvas. Startzeit: 2026-08-27T19:30:25.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.692 und stderr=0.01460648312734278. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: 84ARuGyhQsjgh8umRmBvas. Started at 2026-08-27T19:30:25.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.692 and stderr=0.01460648312734278. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"research:gemini-3.7-flash:21","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.36585365853658536,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.111111111111114,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: KptKXK5nE2Ns2WbiaSSnbc. Startzeit: 2026-08-14T19:59:58.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.36585365853658536 und stderr=0.07615851217559022. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: KptKXK5nE2Ns2WbiaSSnbc. Started at 2026-08-14T19:59:58.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.36585365853658536 and stderr=0.07615851217559022. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gemini-3.7-flash:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.7157894736842105,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":50,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: axMdgNDC8UYYLnZYLHZpF5. Startzeit: 2026-08-14T19:59:58.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.7157894736842105 und stderr=0.026764156649364653. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: axMdgNDC8UYYLnZYLHZpF5. Started at 2026-08-14T19:59:58.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.7157894736842105 and stderr=0.026764156649364653. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":87.84461152882206,"measurements":[{"observationId":"research:gemini-3.7-flash:22","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9482323232323232,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":80.95238095238095,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.7-flash_high. Epoch-Modellbezeichnung: Gemini 3.7 Flash. Anzeigenamenfeld: Gemini 3.7 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: XyVL3VffJ6UboY97vxpohL. Startzeit: 2026-08-14T18:29:12.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9482323232323232 und stderr=0.013456525292246259. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.7-flash_high; Epoch model label: Gemini 3.7 Flash; Display label field: Gemini 3.7 Flash (high); Reasoning setting encoded in the alias: high. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: XyVL3VffJ6UboY97vxpohL. Started at 2026-08-14T18:29:12.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9482323232323232 and stderr=0.013456525292246259. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.94,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 8.773. Standardfehler: 1.488. Kosten je Aufgabe in USD: 0.023122. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 8.773. Standard error: 1.488. Cost per task in USD: 0.023122. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":47.8683966635774,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 47,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 47.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":50,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.7-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1589.905706279442,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":50,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.731 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.7-flash-high. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.7-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,731 published pairwise votes.. Source model ID: lmarena/gemini-3.7-flash-high. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: gemini-3.7-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.731 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 11,731 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.731 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 11,731 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"lmarena:overall:gemini-3.7-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1487.3904875792014,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":77.77777777777777,"participants":37,"methodology":{"de":"Arena-Bewertung aus 26.328 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.7-flash-high. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.7-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 26,328 published pairwise votes.. Source model ID: lmarena/gemini-3.7-flash-high. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: gemini-3.7-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 26.328 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 26,328 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 26.328 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 26,328 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":96.66666666666667,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.7-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1296.8746588285421,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":96.66666666666667,"participants":31,"methodology":{"de":"Arena-Bewertung aus 4.524 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.7-flash-high. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.7-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 4,524 published pairwise votes.. Source model ID: lmarena/gemini-3.7-flash-high. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: gemini-3.7-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 4.524 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 4,524 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 4.524 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 4,524 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":93.06610942249239,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":85.4913294797688,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":94.64285714285714,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 85,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 85.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.96,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.48936170212765,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 6.283. Standardfehler: 0.753. Kosten je Aufgabe in USD: 0.014052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 6.283. Standard error: 0.753. Cost per task in USD: 0.014052. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":71.1864406779661,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":80.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.1864406779661,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 350.439. Standardfehler: 1.763. Kosten je Aufgabe in USD: 1.436459. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 350.439. Standard error: 1.763. Cost per task in USD: 1.436459. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":65.7334730200584,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b2331108-72ed-415a-82d1-188633875bbc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":13.6363636363636,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 13,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b2331108-72ed-415a-82d1-188633875bbc/gemini-3-7-flash/gemini-3-7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.7 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 13.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.7 Flash (High) (Release gemini-3-7-flash, Zeile gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.7 Flash (High) (release gemini-3-7-flash, row gemini-3-7-flash, UUID b2331108-72ed-415a-82d1-188633875bbc, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":77.528,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.67857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 492.202. Standardfehler: 0.649. Kosten je Aufgabe in USD: 1.022219. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 492.202. Standard error: 0.649. Cost per task in USD: 1.022219. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":12.121,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.13513513513514,"participants":38,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 5026.717. Standardfehler: 3.03. Kosten je Aufgabe in USD: 8.705677. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 5026.717. Standard error: 3.03. Cost per task in USD: 8.705677. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":45.16129032258065,"measurements":[{"observationId":"vals-terminal-bench-science:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":7.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.16129032258065,"participants":32,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4825.144. Standardfehler: 3.1. Kosten je Aufgabe in USD: 7.190413. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4825.144. Standard error: 3.1. Cost per task in USD: 7.190413. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":54.83870967741935,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":34.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.83870967741935,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 9520.131. Standardfehler: 4.225. Kosten je Aufgabe in USD: 21.464006. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 9520.131. Standard error: 4.225. Cost per task in USD: 21.464006. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":13.636363636363637,"measurements":[{"observationId":"vals-cyberbench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":43.75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.636363636363637,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 453.761. Standardfehler: 2.21. Kosten je Aufgabe in USD: 2.146121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 453.761. Standard error: 2.21. Cost per task in USD: 2.146121. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":83.05084745762711,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":71.326,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.05084745762711,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 650.914. Standardfehler: 2.262. Kosten je Aufgabe in USD: 5.694001. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 650.914. Standard error: 2.262. Cost per task in USD: 5.694001. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":96.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":59.042,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 181.334. Standardfehler: 0.272. Kosten je Aufgabe in USD: 1.477884. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 181.334. Standard error: 0.272. Cost per task in USD: 1.477884. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-ioi:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":67.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1269.021. Standardfehler: 3.606. Kosten je Aufgabe in USD: 3.398069. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1269.021. Standard error: 3.606. Cost per task in USD: 3.398069. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":94.8051948051948,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":88.652,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.8051948051948,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 14.533. Standardfehler: 0.924. Kosten je Aufgabe in USD: 0.040218. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 14.533. Standard error: 0.924. Cost per task in USD: 0.040218. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":78.68852459016394,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":8.75,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.68852459016394,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 452.101. Standardfehler: 2.148. Kosten je Aufgabe in USD: 2.556743. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 452.101. Standard error: 2.148. Cost per task in USD: 2.556743. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":94.87179487179488,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":87.256,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.87179487179488,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2.227. Standardfehler: 0.416. Kosten je Aufgabe in USD: 0.00409. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2.227. Standard error: 0.416. Cost per task in USD: 0.00409. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":51.666666666666664,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":34.615,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 288.96. Standardfehler: 3.307. Kosten je Aufgabe in USD: 0.760397. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 288.96. Standard error: 3.307. Cost per task in USD: 0.760397. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":89.55223880597015,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":53.39,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.55223880597015,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 9.164. Standardfehler: 2.12. Kosten je Aufgabe in USD: 0.038331. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 9.164. Standard error: 2.12. Cost per task in USD: 0.038331. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":60.294117647058826,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.942,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.294117647058826,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.818. Standardfehler: 2.004. Kosten je Aufgabe in USD: 0.058736. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 16.818. Standard error: 2.004. Cost per task in USD: 0.058736. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":93.50649350649351,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":90.121,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.50649350649351,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.159. Standardfehler: 0.296. Kosten je Aufgabe in USD: 0.010817. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.159. Standard error: 0.296. Cost per task in USD: 0.010817. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":55.76923076923077,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":66.654,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.76923076923077,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4.114. Standardfehler: 0.923. Kosten je Aufgabe in USD: 0.011071. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 4.114. Standard error: 0.923. Cost per task in USD: 0.011071. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1290.211. Standardfehler: 0. Kosten je Aufgabe in USD: 7.130366. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1290.211. Standard error: 0. Cost per task in USD: 7.130366. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":55.12820512820513,"measurements":[{"observationId":"vals-proof-bench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":58,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.12820512820513,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 423.698. Standardfehler: 4.96. Kosten je Aufgabe in USD: 0.558508. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 423.698. Standard error: 4.96. Cost per task in USD: 0.558508. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":68.96551724137932,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":49.232,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.96551724137932,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 9.791. Standardfehler: 3.382. Kosten je Aufgabe in USD: 0.026336. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 9.791. Standard error: 3.382. Cost per task in USD: 0.026336. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":93.10344827586206,"measurements":[{"observationId":"vals-skillsbench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":65.891,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.10344827586206,"participants":30,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 185.651. Standardfehler: 4.554. Kosten je Aufgabe in USD: 1.799454. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 185.651. Standard error: 4.554. Cost per task in USD: 1.799454. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":45.833333333333336,"measurements":[{"observationId":"vals-sre-bench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":4.58,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.833333333333336,"participants":25,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 6606.13. Standardfehler: 1.294. Kosten je Aufgabe in USD: 21.474171. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 6606.13. Standard error: 1.294. Cost per task in USD: 21.474171. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":49.09090909090909,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":22.798,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.09090909090909,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 87.24. Standardfehler: 3.028. Kosten je Aufgabe in USD: 0.483978. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 87.24. Standard error: 3.028. Cost per task in USD: 0.483978. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":72.36842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.734,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.832. Standardfehler: 0.85. Kosten je Aufgabe in USD: 0.018681. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.832. Standard error: 0.85. Cost per task in USD: 0.018681. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":63.888888888888886,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.7-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":70.395,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.888888888888886,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.7-flash. Name des Quellmodells: Gemini 3.7 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1423.494. Standardfehler: 4.837. Kosten je Aufgabe in USD: 4.827643. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.7-flash. Source model name: Gemini 3.7 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1423.494. Standard error: 4.837. Cost per task in USD: 4.827643. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":66.67105876091256,"coveredFamilies":41,"totalFamilies":71,"sourceCount":4,"coverage":0.5509070294784582,"lowerBound":36.72955493415807,"upperBound":81.63885198631226,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":59.31200586509225,"coveredFamilies":9,"totalFamilies":15,"sourceCount":3,"coverage":0.5222222222222225,"lowerBound":30.974047507325967,"upperBound":78.75182528510372,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":86.51031372469114,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":55.613773108730015,"upperBound":91.32805882301574,"provisional":true,"configurationsDiffer":true},"math":{"score":49.77337477337477,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":29.864024864024863,"upperBound":69.86402486402486,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":85.95654571264328,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":85.95654571264328,"upperBound":85.95654571264328,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":34.28301093355761,"upperBound":92.61634426689093,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":24.72222222222222,"upperBound":84.72222222222223,"provisional":true,"configurationsDiffer":true},"finance":{"score":71.38921500741306,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":50.992296433866464,"upperBound":79.56372500529504,"provisional":true,"configurationsDiffer":true},"legal":{"score":75.07566204287515,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":56.30674653215637,"upperBound":81.30674653215637,"provisional":true,"configurationsDiffer":true},"health":{"score":74.92317822651448,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":49.94878548434299,"upperBound":83.28211881767632,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":13.636363636363637,"upperBound":13.636363636363637,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":94.86638804457954,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":47.43319402228977,"upperBound":97.43319402228977,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":34.48275862068966,"upperBound":84.48275862068965,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-3.8-flash","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":61.36363636363637,"measurements":[{"observationId":"research:gemini-3.8-flash:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9888888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.36363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: NeYuqgQyWtFWAHEjQYPm5Z. Startzeit: 2026-09-02T17:15:36.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9888888888888889 und stderr=0.008724007409153888. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: NeYuqgQyWtFWAHEjQYPm5Z. Started at 2026-09-02T17:15:36.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9888888888888889 and stderr=0.008724007409153888. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":67.10526315789474,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":46.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.10526315789474,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 46,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 46.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1202.59,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":77.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":81.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":77.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 81,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 81.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":59.93009432118242,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 59,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 59.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":36.3585,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.666666666666668,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 36,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 36.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":21,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 21,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 21.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1436.3,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":97.14285714285714,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":52.5423728813559,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.14285714285714,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 52,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 52.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":29.55,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":76.19047619047619,"measurements":[{"observationId":"research:gemini-3.8-flash:18","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.58,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":76.19047619047619,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: Hpyy6CpmXmd5q7Zbdq63CR. Startzeit: 2026-10-13T12:39:05.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.58 und stderr=0.04960449637488583. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: Hpyy6CpmXmd5q7Zbdq63CR. Started at 2026-10-13T12:39:05.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.58 and stderr=0.04960449637488583. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"research:gemini-3.8-flash:19","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.697,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":83.33333333333333,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: gZ4bwozTKtpmZTqnqX3a33. Startzeit: 2026-09-02T17:15:36.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.697 und stderr=0.014539683710535193. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: gZ4bwozTKtpmZTqnqX3a33. Started at 2026-09-02T17:15:36.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.697 and stderr=0.014539683710535193. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":30.176767676767675,"measurements":[{"observationId":"research:gemini-3.8-flash:16","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.21951219512195122,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":19.444444444444443,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: Lgy52ZT4YpFrUqqKVu2qUs. Startzeit: 2026-09-02T17:15:36.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.21951219512195122 und stderr=0.06544589202438408. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: Lgy52ZT4YpFrUqqKVu2qUs. Started at 2026-09-02T17:15:36.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.21951219512195122 and stderr=0.06544589202438408. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gemini-3.8-flash:15","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.6842105263157895,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":40.90909090909091,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: Utgs6DuaQDDdi98ZWH22qL. Startzeit: 2026-09-02T17:15:36.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.6842105263157895 und stderr=0.027582557383453006. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: Utgs6DuaQDDdi98ZWH22qL. Started at 2026-09-02T17:15:36.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.6842105263157895 and stderr=0.027582557383453006. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":92.07393483709274,"measurements":[{"observationId":"research:gemini-3.8-flash:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9539141414141414,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":88.0952380952381,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-3.8-flash_high. Epoch-Modellbezeichnung: Gemini 3.8 Flash. Anzeigenamenfeld: Gemini 3.8 Flash (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: Br3mjs2GrYcCwX9AMwcihU. Startzeit: 2026-09-02T17:15:36.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9539141414141414 und stderr=0.01401136318210652. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-3.8-flash_high; Epoch model label: Gemini 3.8 Flash; Display label field: Gemini 3.8 Flash (high); Reasoning setting encoded in the alias: high. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: Br3mjs2GrYcCwX9AMwcihU. Started at 2026-09-02T17:15:36.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9539141414141414 and stderr=0.01401136318210652. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":94.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 20.18. Standardfehler: 1.477. Kosten je Aufgabe in USD: 0.056302. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 20.18. Standard error: 1.477. Cost per task in USD: 0.056302. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":86.58536585365853,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":47.8220574606117,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 47,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 47.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":47.22222222222222,"measurements":[{"observationId":"lmarena:webdev:overall:gemini-3.8-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1584.7701774775737,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":47.22222222222222,"participants":37,"methodology":{"de":"Arena-Bewertung aus 10.077 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.8-flash-high. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.8-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,077 published pairwise votes.. Source model ID: lmarena/gemini-3.8-flash-high. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: gemini-3.8-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.077 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 10,077 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.077 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 10,077 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"lmarena:overall:gemini-3.8-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1496.3187176758736,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":88.88888888888889,"participants":37,"methodology":{"de":"Arena-Bewertung aus 35.670 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.8-flash-high. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.8-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 35,670 published pairwise votes.. Source model ID: lmarena/gemini-3.8-flash-high. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: gemini-3.8-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 35.670 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 35,670 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 35.670 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 35,670 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gemini-3.8-flash-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1289.1291293859153,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":83.33333333333333,"participants":31,"methodology":{"de":"Arena-Bewertung aus 3.792 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gemini-3.8-flash-high. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: gemini-3.8-flash-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 3,792 published pairwise votes.. Source model ID: lmarena/gemini-3.8-flash-high. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: gemini-3.8-flash-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 3.792 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 3,792 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 3.792 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 3,792 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":95.02279635258358,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":85.606936416185,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.42857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 85,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 85.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":89.075,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.61702127659575,"participants":48,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 12.028. Standardfehler: 0.75. Kosten je Aufgabe in USD: 0.004472. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 12.028. Standard error: 0.75. Cost per task in USD: 0.004472. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":69.49152542372882,"measurements":[{"observationId":"vals-swebench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":80,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.49152542372882,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 683.446. Standardfehler: 1.791. Kosten je Aufgabe in USD: 2.1911. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 683.446. Standard error: 1.791. Cost per task in USD: 2.1911. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":73.78872775214239,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":19.6969696969697,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 19,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 19.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":81.273,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.5,"participants":57,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 554.797. Standardfehler: 0.375. Kosten je Aufgabe in USD: 1.544972. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 554.797. Standard error: 0.375. Cost per task in USD: 1.544972. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":19.697,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.2972972972973,"participants":38,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 6427.575. Standardfehler: 2.314. Kosten je Aufgabe in USD: 8.64749. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 6427.575. Standard error: 2.314. Cost per task in USD: 8.64749. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":52.45441795231417,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":10,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.52173913043478,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Modellstand: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 10,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/8a999846-4c1d-4ce7-a8b7-1310a7166fd7/gemini-3-8-flash/gemini-3-8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Model snapshot: Gemini 3.8 Flash (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 10.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemini 3.8 Flash (High) (Release gemini-3-8-flash, Zeile gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Gemini 3.8 Flash (High) (release gemini-3-8-flash, row gemini-3-8-flash, UUID 8a999846-4c1d-4ce7-a8b7-1310a7166fd7, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":8.571,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.38709677419355,"participants":32,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3311.435. Standardfehler: 3.37. Kosten je Aufgabe in USD: 5.643008. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3311.435. Standard error: 3.37. Cost per task in USD: 5.643008. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":21.05263157894737,"measurements":[{"observationId":"vals-biomysterybench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":62.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.05263157894737,"participants":20,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 326.755. Standardfehler: 1.697. Kosten je Aufgabe in USD: 1.5539. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 326.755. Standard error: 1.697. Cost per task in USD: 1.5539. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":58.064516129032256,"measurements":[{"observationId":"vals-code-migration:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":36.546,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.064516129032256,"participants":63,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 9443.316. Standardfehler: 4.184. Kosten je Aufgabe in USD: 18.491664. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 9443.316. Standard error: 4.184. Cost per task in USD: 18.491664. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":13.636363636363637,"measurements":[{"observationId":"vals-cyberbench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":43.75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.636363636363637,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 588.848. Standardfehler: 2.21. Kosten je Aufgabe in USD: 1.305189. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 588.848. Standard error: 2.21. Cost per task in USD: 1.305189. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":89.83050847457628,"measurements":[{"observationId":"vals-emb:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":72.196,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.83050847457628,"participants":60,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 769.197. Standardfehler: 2.42. Kosten je Aufgabe in USD: 8.243862. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 769.197. Standard error: 2.42. Cost per task in USD: 8.243862. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"vals-finance-agent-v2:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":61.435,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 200.952. Standardfehler: 0.128. Kosten je Aufgabe in USD: 2.000457. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 200.952. Standard error: 0.128. Cost per task in USD: 2.000457. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":51.515151515151516,"measurements":[{"observationId":"vals-ioi:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":56.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.515151515151516,"participants":34,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 860.465. Standardfehler: 2.257. Kosten je Aufgabe in USD: 3.976052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 860.465. Standard error: 2.257. Cost per task in USD: 3.976052. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":97.40259740259741,"measurements":[{"observationId":"vals-lcb:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":89.484,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.40259740259741,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 24.07. Standardfehler: 0.897. Kosten je Aufgabe in USD: 0.07124. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 24.07. Standard error: 0.897. Cost per task in USD: 0.07124. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":81.9672131147541,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":10,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.9672131147541,"participants":62,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1760.871. Standardfehler: 2.415. Kosten je Aufgabe in USD: 3.656207. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1760.871. Standard error: 2.415. Cost per task in USD: 3.656207. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":93.58974358974359,"measurements":[{"observationId":"vals-legal-bench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.993,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.58974358974359,"participants":79,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3.32. Standardfehler: 0.432. Kosten je Aufgabe in USD: 0.008109. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3.32. Standard error: 0.432. Cost per task in USD: 0.008109. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":63.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":38.942,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 324.318. Standardfehler: 3.389. Kosten je Aufgabe in USD: 1.630498. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 324.318. Standard error: 3.389. Cost per task in USD: 1.630498. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":67.16417910447761,"measurements":[{"observationId":"vals-medcode:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":48.135,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.16417910447761,"participants":68,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 42.892. Standardfehler: 2.18. Kosten je Aufgabe in USD: 0.0177. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 42.892. Standard error: 2.18. Cost per task in USD: 0.0177. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":64.70588235294117,"measurements":[{"observationId":"vals-medscribe:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":84.496,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.70588235294117,"participants":69,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 21.145. Standardfehler: 1.943. Kosten je Aufgabe in USD: 0.025238. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 21.145. Standard error: 1.943. Cost per task in USD: 0.025238. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":94.8051948051948,"measurements":[{"observationId":"vals-mmlu-pro:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":90.221,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.8051948051948,"participants":78,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.82. Standardfehler: 0.295. Kosten je Aufgabe in USD: 0.022457. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.82. Standard error: 0.295. Cost per task in USD: 0.022457. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.342,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.46153846153846,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 14.794. Standardfehler: 0.943. Kosten je Aufgabe in USD: 0.046973. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 14.794. Standard error: 0.943. Cost per task in USD: 0.046973. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":70,"measurements":[{"observationId":"vals-mysterymechanism:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":36.486,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70,"participants":21,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 322.237. Standardfehler: 3.238. Kosten je Aufgabe in USD: 1.731041. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 322.237. Standard error: 3.238. Cost per task in USD: 1.731041. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":73.46938775510205,"measurements":[{"observationId":"vals-programbench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":1,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.46938775510205,"participants":50,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2799.268. Standardfehler: 0.705. Kosten je Aufgabe in USD: 10.928509. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 2799.268. Standard error: 0.705. Cost per task in USD: 10.928509. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":41.02564102564103,"measurements":[{"observationId":"vals-proof-bench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":48,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.02564102564103,"participants":40,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 368.648. Standardfehler: 5.021. Kosten je Aufgabe in USD: 0.602872. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 368.648. Standard error: 5.021. Cost per task in USD: 0.602872. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":15.517241379310345,"measurements":[{"observationId":"vals-sage:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":35.064,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.517241379310345,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 28.639. Standardfehler: 3.364. Kosten je Aufgabe in USD: 0.008954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 28.639. Standard error: 3.364. Cost per task in USD: 0.008954. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":68.96551724137932,"measurements":[{"observationId":"vals-skillsbench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":57.985,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.96551724137932,"participants":30,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 235.201. Standardfehler: 4.422. Kosten je Aufgabe in USD: 2.336944. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 235.201. Standard error: 4.422. Cost per task in USD: 2.336944. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":60.526315789473685,"measurements":[{"observationId":"vals-snap:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":65.291,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.526315789473685,"participants":39,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 201.419. Standardfehler: 1.238. Kosten je Aufgabe in USD: 1.011271. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 201.419. Standard error: 1.238. Cost per task in USD: 1.011271. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":62.5,"measurements":[{"observationId":"vals-sre-bench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":11.069,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":25,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 8853.856. Standardfehler: 1.942. Kosten je Aufgabe in USD: 27.459572. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 8853.856. Standard error: 1.942. Cost per task in USD: 27.459572. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"vals-swebench-multimodal:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":40.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":14,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1338.303. Standardfehler: 2.246. Kosten je Aufgabe in USD: 3.688847. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 1338.303. Standard error: 2.246. Cost per task in USD: 3.688847. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":78.18181818181819,"measurements":[{"observationId":"vals-tax-agent-bench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":32.124,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.18181818181819,"participants":56,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 178.329. Standardfehler: 3.37. Kosten je Aufgabe in USD: 0.857214. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 178.329. Standard error: 3.37. Cost per task in USD: 0.857214. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":68.42105263157895,"measurements":[{"observationId":"vals-tax-eval-v2:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.448,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.42105263157895,"participants":77,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 8.759. Standardfehler: 0.848. Kosten je Aufgabe in USD: 0.023535. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 8.759. Standard error: 0.848. Cost per task in USD: 0.023535. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":52.38095238095238,"measurements":[{"observationId":"vals-vcb-1-100:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":18.772,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.38095238095238,"participants":22,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3931.352. Standardfehler: 3.313. Kosten je Aufgabe in USD: 18.906982. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 3931.352. Standard error: 3.313. Cost per task in USD: 18.906982. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":73.61111111111111,"measurements":[{"observationId":"vals-vibe-code-bench:overall:google/gemini-3.8-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":78.651,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.61111111111111,"participants":73,"methodology":{"de":"Kennung des Quellmodells: google/gemini-3.8-flash. Name des Quellmodells: Gemini 3.8 Flash. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 519.104. Standardfehler: 3.884. Kosten je Aufgabe in USD: 6.865141. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemini-3.8-flash. Source model name: Gemini 3.8 Flash. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 519.104. Standard error: 3.884. Cost per task in USD: 6.865141. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":61.85934097992619,"coveredFamilies":48,"totalFamilies":71,"sourceCount":4,"coverage":0.63781179138322,"lowerBound":39.45461708419215,"upperBound":75.67343794587015,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":57.5382631221226,"coveredFamilies":12,"totalFamilies":15,"sourceCount":3,"coverage":0.7555555555555555,"lowerBound":43.47335435893708,"upperBound":67.91779880338153,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":81.83653578390421,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":52.60920157536699,"upperBound":88.32348728965272,"provisional":true,"configurationsDiffer":true},"math":{"score":44.18868168868168,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":26.51320901320901,"upperBound":66.513209013209,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":89.86731425755816,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":89.86731425755816,"upperBound":89.86731425755816,"provisional":true,"configurationsDiffer":true},"agentic":{"score":75.86080950905439,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":37.930404754527196,"upperBound":87.93040475452719,"provisional":true,"configurationsDiffer":true},"general":{"score":60.610380116959064,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":48.48830409356725,"upperBound":68.48830409356725,"provisional":true,"configurationsDiffer":true},"finance":{"score":74.97898354990238,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":53.55641682135884,"upperBound":82.12784539278742,"provisional":true,"configurationsDiffer":true},"legal":{"score":79.630096679277,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":59.722572509457756,"upperBound":84.72257250945776,"provisional":true,"configurationsDiffer":true},"health":{"score":65.93503072870939,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":43.95668715247293,"upperBound":77.29002048580627,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":13.636363636363637,"upperBound":13.636363636363637,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":89.17806484295845,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":44.589032421479224,"upperBound":94.58903242147923,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":30.263157894736842,"upperBound":80.26315789473685,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":7.758620689655172,"upperBound":57.758620689655174,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemini-4-argon","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"research:gemini-4-argon:9","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":1,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-4-argon_high. Epoch-Modellbezeichnung: Gemini 4 Argon. Anzeigenamenfeld: Gemini 4 Argon (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: 43f36RTDoJGxXWQDVvTLJj. Startzeit: 2026-10-13T17:14:57.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=1.0 und stderr=0.0. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-4-argon_high; Epoch model label: Gemini 4 Argon; Display label field: Gemini 4 Argon (high); Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: 43f36RTDoJGxXWQDVvTLJj. Started at 2026-10-13T17:14:57.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=1.0 and stderr=0.0. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":80.95238095238095,"measurements":[{"observationId":"research:gemini-4-argon:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.59,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":80.95238095238095,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-4-argon_high. Epoch-Modellbezeichnung: Gemini 4 Argon. Anzeigenamenfeld: Gemini 4 Argon (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: GJtYeC7ZV4fYgN3wu4rRvq. Startzeit: 2026-10-13T17:15:18.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.59 und stderr=0.04943110704237103. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-4-argon_high; Epoch model label: Gemini 4 Argon; Display label field: Gemini 4 Argon (high); Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: GJtYeC7ZV4fYgN3wu4rRvq. Started at 2026-10-13T17:15:18.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.59 and stderr=0.04943110704237103. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"research:gemini-4-argon:8","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.76,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":100,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-4-argon_high. Epoch-Modellbezeichnung: Gemini 4 Argon. Anzeigenamenfeld: Gemini 4 Argon (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: iq4Y9tXpiyLP25yotySxJW. Startzeit: 2026-10-13T17:15:14.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.76 und stderr=0.013512312258920845. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-4-argon_high; Epoch model label: Gemini 4 Argon; Display label field: Gemini 4 Argon (high); Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: iq4Y9tXpiyLP25yotySxJW. Started at 2026-10-13T17:15:14.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.76 and stderr=0.013512312258920845. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":0.5,"score":81.81818181818181,"measurements":[{"observationId":"research:gemini-4-argon:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.8596491228070176,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":81.81818181818181,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-4-argon_high. Epoch-Modellbezeichnung: Gemini 4 Argon. Anzeigenamenfeld: Gemini 4 Argon (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: N8daAyKM6Sah39YBcPRi4W. Startzeit: 2026-10-13T17:14:57.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.8596491228070176 und stderr=0.020611471958164294. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-4-argon_high; Epoch model label: Gemini 4 Argon; Display label field: Gemini 4 Argon (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: N8daAyKM6Sah39YBcPRi4W. Started at 2026-10-13T17:14:57.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.8596491228070176 and stderr=0.020611471958164294. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":100,"measurements":[{"observationId":"research:gemini-4-argon:6","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9640151515151515,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":100,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gemini-4-argon_high. Epoch-Modellbezeichnung: Gemini 4 Argon. Anzeigenamenfeld: Gemini 4 Argon (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: QEouTvczNsxgkR4i7i8wBF. Startzeit: 2026-10-13T17:15:08.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9640151515151515 und stderr=0.011918129589602694. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: gemini-4-argon_high; Epoch model label: Gemini 4 Argon; Display label field: Gemini 4 Argon (high); Reasoning setting encoded in the alias: high. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: QEouTvczNsxgkR4i7i8wBF. Started at 2026-10-13T17:15:08.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9640151515151515 and stderr=0.011918129589602694. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":5,"totalFamilies":71,"sourceCount":1,"coverage":0.06054421768707484,"lowerBound":5.600318049297642,"upperBound":99.54589628059016,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2142857142857143,"lowerBound":18.707482993197278,"upperBound":97.27891156462584,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.30000000000000004,"lowerBound":26.363636363636363,"upperBound":96.36363636363636,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":33.333333333333336,"upperBound":100,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemma-2-27b","contributions":[{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":6.410256410256411,"measurements":[{"observationId":"vals-legal-bench:overall:together/google/gemma-2-27b-it","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":67.978,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.410256410256411,"participants":79,"methodology":{"de":"Kennung des Quellmodells: together/google/gemma-2-27b-it. Name des Quellmodells: Gemma 2 27B. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 0.887. Standardfehler: 0.651. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/google/gemma-2-27b-it. Source model name: Gemma 2 27B. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Maximum output tokens: 4096. Latency in seconds: 0.887. Standard error: 0.651. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 4096","en":"Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 4096","en":"Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.01785714285714286,"lowerBound":0.11446886446886448,"upperBound":98.32875457875457,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":1.6025641025641026,"upperBound":76.6025641025641,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemma-3-12b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gemma-3-27b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gemma-3-4b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gemma-4-26b-a4b","contributions":[{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":5.714285714285714,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:6e1b44ff-c227-496b-aef4-19b70cd18c76","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":23.6346516007533,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":5.714285714285714,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 26B A4B (Reasoning) (Release gemma-4-26b-a4b, Zeile gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/6e1b44ff-c227-496b-aef4-19b70cd18c76/gemma-4-26b-a4b/gemma-4-26b-a4b. Name des Quellmodells: Gemma 4 26B A4B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 26B A4B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 23,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 26B A4B (Reasoning) (release gemma-4-26b-a4b, row gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/6e1b44ff-c227-496b-aef4-19b70cd18c76/gemma-4-26b-a4b/gemma-4-26b-a4b. Source model name: Gemma 4 26B A4B. Provider reported by source: Google. Model snapshot: Gemma 4 26B A4B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 23.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 26B A4B (Reasoning) (Release gemma-4-26b-a4b, Zeile gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 26B A4B (Reasoning) (release gemma-4-26b-a4b, row gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 26B A4B (Reasoning) (Release gemma-4-26b-a4b, Zeile gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 26B A4B (Reasoning) (release gemma-4-26b-a4b, row gemma-4-26b-a4b, UUID 6e1b44ff-c227-496b-aef4-19b70cd18c76, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.005952380952380953,"lowerBound":0.034013605442176874,"upperBound":99.4387755102041,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":0.47619047619047616,"upperBound":92.14285714285714,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gemma-4-31b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":367.87,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":69.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 69,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 69.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":31.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":4.933063448681232,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 4,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 4.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":31.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 6,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 6.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":619.79,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":37.142857142857146,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":37.288135593220304,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.142857142857146,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 37,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 37.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":-47.93333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":23.6329935125116,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 23,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 23.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":23.976480836236934,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:cd26a386-4873-46ff-b853-d239050025a2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=1736:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Modellstand: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd26a386-4873-46ff-b853-d239050025a2/gemma-4-31b/gemma-4-31b. Source model name: Gemma 4 31B. Provider reported by source: Google. Model snapshot: Gemma 4 31B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Gemma 4 31B (Reasoning) (Release gemma-4-31b, Zeile gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Gemma 4 31B (Reasoning) (release gemma-4-31b, row gemma-4-31b, UUID cd26a386-4873-46ff-b853-d239050025a2, reasoning=true, source reasoning-token configuration budget=1736, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:google/gemma-4-31b-it","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":39.326,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":43,"methodology":{"de":"Kennung des Quellmodells: google/gemma-4-31b-it. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 855.298. Standardfehler: 5.207. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemma-4-31b-it. Source model name: Gemma 4 31B. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 855.298. Standard error: 5.207. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":21.153846153846153,"measurements":[{"observationId":"vals-mortgage-tax:overall:google/gemma-4-31b-it","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":61.368,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.153846153846153,"participants":53,"methodology":{"de":"Kennung des Quellmodells: google/gemma-4-31b-it. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 41.928. Standardfehler: 0.967. Kosten je Aufgabe in USD: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemma-4-31b-it. Source model name: Gemma 4 31B. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 41.928. Standard error: 0.967. Cost per task in USD: 0. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":98.27586206896552,"measurements":[{"observationId":"vals-sage:overall:google/gemma-4-31b-it","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":55.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.27586206896552,"participants":59,"methodology":{"de":"Kennung des Quellmodells: google/gemma-4-31b-it. Name des Quellmodells: Gemma 4 31B. Anbieter laut Quelle: Google. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 93.058. Standardfehler: 3.411. Kosten je Aufgabe in USD: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: google/gemma-4-31b-it. Source model name: Gemma 4 31B. Provider reported by source: Google. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 93.058. Standard error: 3.411. Cost per task in USD: 0. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":11,"totalFamilies":71,"sourceCount":2,"coverage":0.12222222222222223,"lowerBound":5.560783283359279,"upperBound":93.33856106113706,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":2,"coverage":0.04444444444444445,"lowerBound":1.0656213704994195,"upperBound":96.62117692605496,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":10.452961672473867,"upperBound":74.73867595818815,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":4.471544715447155,"upperBound":71.13821138211382,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.700929152148664,"upperBound":76.36759581881533,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":3.021978021978022,"upperBound":88.73626373626374,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":49.13793103448276,"upperBound":99.13793103448276,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-4.5","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":37.68115942028985,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:1cf439b8-0cfd-47b2-9de2-9a2157e6762b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":44.0816326530612,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.68115942028985,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GLM-4.5 (Reasoning) (Release glm-4-5, Zeile glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/1cf439b8-0cfd-47b2-9de2-9a2157e6762b/glm-4-5/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Z AI. Modellstand: GLM-4.5 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 44,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-4.5 (Reasoning) (release glm-4-5, row glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/1cf439b8-0cfd-47b2-9de2-9a2157e6762b/glm-4-5/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Z AI. Model snapshot: GLM-4.5 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 44.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-4.5 (Reasoning) (Release glm-4-5, Zeile glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-4.5 (Reasoning) (release glm-4-5, row glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-4.5 (Reasoning) (Release glm-4-5, Zeile glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-4.5 (Reasoning) (release glm-4-5, row glm-4.5, UUID 1cf439b8-0cfd-47b2-9de2-9a2157e6762b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":21.05263157894737,"measurements":[{"observationId":"vals-gpqa:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":72.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 704.224. Standardfehler: 2.262. Kosten je Aufgabe in USD: 0.019454. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 704.224. Standard error: 2.262. Cost per task in USD: 0.019454. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":44.5945945945946,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.956,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.5945945945946,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 95.244. Standardfehler: 0.959. Kosten je Aufgabe in USD: 0.03783. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 95.244. Standard error: 0.959. Cost per task in USD: 0.03783. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":67.446,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 391.268. Standardfehler: 1.131. Kosten je Aufgabe in USD: 0.018278. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 391.268. Standard error: 1.131. Cost per task in USD: 0.018278. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":75.627,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.384615384615385,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 16.402. Standardfehler: 0.511. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 16.402. Standard error: 0.511. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":68.18181818181819,"measurements":[{"observationId":"vals-math500:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.18181818181819,"participants":23,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Latenz in Sekunden: 84.928. Standardfehler: 1.062. Kosten je Aufgabe in USD: 0.012744. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Latency in seconds: 84.928. Standard error: 1.062. Cost per task in USD: 0.012744. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":41.02564102564103,"measurements":[{"observationId":"vals-medqa:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":89.975,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.02564102564103,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 59.609. Standardfehler: 0.274. Kosten je Aufgabe in USD: 0.003386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 59.609. Standard error: 0.274. Cost per task in USD: 0.003386. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":55.172413793103445,"measurements":[{"observationId":"vals-mgsm:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":90.836,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.172413793103445,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 89.549. Standardfehler: 1.791. Kosten je Aufgabe in USD: 0.006319. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 89.549. Standard error: 1.791. Cost per task in USD: 0.006319. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":29.87012987012987,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":81.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.87012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 136.72. Standardfehler: 0.378. Kosten je Aufgabe in USD: 0.009757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 136.72. Standard error: 0.378. Cost per task in USD: 0.009757. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":47.36842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.404,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.5. Name des Quellmodells: GLM-4.5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920. Latenz in Sekunden: 129.388. Standardfehler: 0.872. Kosten je Aufgabe in USD: 0.009559. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.5. Source model name: GLM-4.5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 1. Maximum output tokens: 81920. Latency in seconds: 129.388. Standard error: 0.872. Cost per task in USD: 0.009559. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 81920","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 81920"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":2,"coverage":0.1386054421768708,"lowerBound":5.395615002259049,"upperBound":91.53507078457197,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":1.2121212121212126,"upperBound":94.54545454545455,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.5037593984962407,"upperBound":94.3609022556391,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":24.670846394984327,"upperBound":84.67084639498432,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.956709956709958,"upperBound":76.62337662337663,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":7.536231884057971,"upperBound":87.53623188405797,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":13.137573663889455,"upperBound":84.56614509246089,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":3.846153846153846,"upperBound":78.84615384615384,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":13.675213675213676,"upperBound":80.34188034188034,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-4.5-air","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":21.73913043478261,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:5d303dc9-c027-401f-9803-4e9aa3331007","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":37.551020408163296,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.73913043478261,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GLM-4.5-Air (Release glm-4-5-air, Zeile glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/5d303dc9-c027-401f-9803-4e9aa3331007/glm-4-5-air/glm-4-5-air. Name des Quellmodells: GLM-4.5 Air. Anbieter laut Quelle: Z AI. Modellstand: GLM-4.5-Air. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 37,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-4.5-Air (release glm-4-5-air, row glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/5d303dc9-c027-401f-9803-4e9aa3331007/glm-4-5-air/glm-4-5-air. Source model name: GLM-4.5 Air. Provider reported by source: Z AI. Model snapshot: GLM-4.5-Air. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 37.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-4.5-Air (Release glm-4-5-air, Zeile glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-4.5-Air (release glm-4-5-air, row glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-4.5-Air (Release glm-4-5-air, Zeile glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-4.5-Air (release glm-4-5-air, row glm-4-5-air, UUID 5d303dc9-c027-401f-9803-4e9aa3331007, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.31055900621118016,"upperBound":98.88198757763976,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.3478260869565215,"upperBound":84.34782608695652,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-4.5v","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"glm-4.6","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":24.342105263157894,"measurements":[{"observationId":"vals-gpqa:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":74.495,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.342105263157894,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 474.834. Standardfehler: 2.197. Kosten je Aufgabe in USD: 0.01541. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 474.834. Standard error: 2.197. Cost per task in USD: 0.01541. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":23.80952380952381,"measurements":[{"observationId":"vals-terminal-bench-2:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":28.09,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.80952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 576.085. Standardfehler: 4.791. Kosten je Aufgabe in USD: 0.14355. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 576.085. Standard error: 4.791. Cost per task in USD: 0.14355. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":21.62162162162162,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":56.838,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.62162162162162,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 78.061. Standardfehler: 0.97. Kosten je Aufgabe in USD: 0.04461. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.6. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 78.061. Standard error: 0.97. Cost per task in USD: 0.04461. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":37.66233766233766,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.036,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.66233766233766,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 236.176. Standardfehler: 1.082. Kosten je Aufgabe in USD: 0.026909. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 236.176. Standard error: 1.082. Cost per task in USD: 0.026909. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":28.205128205128204,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.608,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.205128205128204,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 29.819. Standardfehler: 0.522. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 29.819. Standard error: 0.522. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":56.41025641025641,"measurements":[{"observationId":"vals-medqa:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":92.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.41025641025641,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 28.734. Standardfehler: 0.244. Kosten je Aufgabe in USD: 0.004252. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 28.734. Standard error: 0.244. Cost per task in USD: 0.004252. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":48.275862068965516,"measurements":[{"observationId":"vals-mgsm:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":89.746,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.275862068965516,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 34.307. Standardfehler: 1.803. Kosten je Aufgabe in USD: 0.004302. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 34.307. Standard error: 1.803. Cost per task in USD: 0.004302. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":32.467532467532465,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":82.203,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.467532467532465,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 46.996. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.006572. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 46.996. Standard error: 0.375. Cost per task in USD: 0.006572. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 1. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"vals-poker-agent:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":945.7558288256936,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":9,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 42.55249388614798. Standardfehler: 0. Kosten je Aufgabe in USD: 0.019527262827635273. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 42.55249388614798. Standard error: 0. Cost per task in USD: 0.019527262827635273. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":9.210526315789474,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":66.235,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.210526315789474,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 64.149. Standardfehler: 0.808. Kosten je Aufgabe in USD: 0.00928. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 64.149. Standard error: 0.808. Cost per task in USD: 0.00928. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":8.333333333333334,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":3.09,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.333333333333334,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.6. Name des Quellmodells: GLM-4.6. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880. Latenz in Sekunden: 10002.446. Standardfehler: 1.162. Kosten je Aufgabe in USD: 10.845475. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.6. Source model name: GLM-4.6. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.6. Top P: 1. Maximum output tokens: 122880. Latency in seconds: 10002.446. Standard error: 1.162. Cost per task in USD: 10.845475. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Harness: OpenHands. Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 1. Maximale Ausgabetoken: 122880","en":"Harness: OpenHands. Temperature: 0.6. Top P: 1. Maximum output tokens: 122880"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":11,"totalFamilies":71,"sourceCount":1,"coverage":0.12154195011337869,"lowerBound":3.9861876277808355,"upperBound":91.83199261644297,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":1,"coverage":0.14444444444444451,"lowerBound":3.330928330928332,"upperBound":88.88648388648389,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.7387218045112782,"upperBound":94.59586466165413,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":9.655172413793103,"upperBound":89.65517241379311,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":10.822510822510822,"upperBound":77.4891774891775,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":0,"upperBound":91.66666666666666,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":4.404592562487299,"upperBound":75.83316399105873,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":7.051282051282051,"upperBound":82.05128205128204,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":18.803418803418804,"upperBound":85.47008547008548,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-4.6v","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"glm-4.7","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":35.526315789473685,"measurements":[{"observationId":"vals-gpqa:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":80.05,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 211.079. Standardfehler: 2.006. Kosten je Aufgabe in USD: 0.024866. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 211.079. Standard error: 2.006. Cost per task in USD: 0.024866. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":22.033898305084747,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":69.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.033898305084747,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 365.799. Standardfehler: 2.063. Kosten je Aufgabe in USD: 0.203695. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 1. Maximum output tokens: 16384. Latency in seconds: 365.799. Standard error: 2.063. Cost per task in USD: 0.203695. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 16384","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":40.476190476190474,"measurements":[{"observationId":"vals-terminal-bench-2:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":38.202,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.476190476190474,"participants":43,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 777.512. Standardfehler: 5.18. Kosten je Aufgabe in USD: 0.183983. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 777.512. Standard error: 5.18. Cost per task in USD: 0.183983. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":6.756756756756757,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":46.387,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.756756756756757,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 66.101. Standardfehler: 0.899. Kosten je Aufgabe in USD: 0.039082. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 66.101. Standard error: 0.899. Cost per task in USD: 0.039082. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":51.94805194805195,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.234,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.94805194805195,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 393.276. Standardfehler: 0.993. Kosten je Aufgabe in USD: 0.035469. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 393.276. Standard error: 0.993. Cost per task in USD: 0.035469. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":52.56410256410256,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.358,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.56410256410256,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 30.675. Standardfehler: 0.473. Kosten je Aufgabe in USD: 0.002348. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 30.675. Standard error: 0.473. Cost per task in USD: 0.002348. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":13.432835820895523,"measurements":[{"observationId":"vals-medcode:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":32.772,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.432835820895523,"participants":68,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 123.441. Standardfehler: 1.996. Kosten je Aufgabe in USD: 0.00671. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 123.441. Standard error: 1.996. Cost per task in USD: 0.00671. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"vals-medqa:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":93.742,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.23076923076923,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 66.008. Standardfehler: 0.221. Kosten je Aufgabe in USD: 0.00487. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 66.008. Standard error: 0.221. Cost per task in USD: 0.00487. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":8.823529411764707,"measurements":[{"observationId":"vals-medscribe:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":68.629,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.823529411764707,"participants":69,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 167.479. Standardfehler: 2.123. Kosten je Aufgabe in USD: 0.019082. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 167.479. Standard error: 2.123. Cost per task in USD: 0.019082. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":34.48275862068966,"measurements":[{"observationId":"vals-mgsm:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":88.182,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.48275862068966,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 63.175. Standardfehler: 1.828. Kosten je Aufgabe in USD: 0.004315. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 63.175. Standard error: 1.828. Cost per task in USD: 0.004315. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":35.064935064935064,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":82.736,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.064935064935064,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 104.451. Standardfehler: 0.371. Kosten je Aufgabe in USD: 0.00798. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 104.451. Standard error: 0.371. Cost per task in USD: 0.00798. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":19.736842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":68.766,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.736842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-4.7. Name des Quellmodells: GLM-4.7. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 187.181. Standardfehler: 0.921. Kosten je Aufgabe in USD: 0.013873. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-4.7. Source model name: GLM-4.7. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 1. Maximum output tokens: 128000. Latency in seconds: 187.181. Standard error: 0.921. Cost per task in USD: 0.013873. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":1,"coverage":0.160827664399093,"lowerBound":5.227956896675284,"upperBound":89.14519045676597,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":1,"coverage":0.11111111111111113,"lowerBound":4.647402189775073,"upperBound":93.53629107866396,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":2.537593984962406,"upperBound":95.39473684210526,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.8965517241379315,"upperBound":86.89655172413794,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":11.688311688311687,"upperBound":78.35497835497836,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":3.7847998374314162,"upperBound":75.21337126600285,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":13.14102564102564,"upperBound":88.14102564102564,"provisional":true,"configurationsDiffer":true},"health":{"score":30.49571148780982,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":30.49571148780982,"upperBound":30.49571148780982,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-4.7-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"glm-5","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:40663ad2-b218-471e-bdd4-a1e0c2360e2b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=3105:suffix=Reasoning","retrievedAt":"2026-10-19","value":14.454277286135694,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.076923076923077,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5 (Reasoning) (Release glm-5, Zeile glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/40663ad2-b218-471e-bdd4-a1e0c2360e2b/glm-5/glm-5. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Z AI. Modellstand: GLM-5 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 14,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5 (Reasoning) (release glm-5, row glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/40663ad2-b218-471e-bdd4-a1e0c2360e2b/glm-5/glm-5. Source model name: GLM-5. Provider reported by source: Z AI. Model snapshot: GLM-5 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 14.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5 (Reasoning) (Release glm-5, Zeile glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5 (Reasoning) (release glm-5, row glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5 (Reasoning) (Release glm-5, Zeile glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5 (Reasoning) (release glm-5, row glm-5, UUID 40663ad2-b218-471e-bdd4-a1e0c2360e2b, reasoning=true, source reasoning-token configuration budget=3105, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":44.73684210526316,"measurements":[{"observationId":"vals-gpqa:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":83.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 242.523. Standardfehler: 1.873. Kosten je Aufgabe in USD: 0.034091. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 242.523. Standard error: 1.873. Cost per task in USD: 0.034091. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":31.35593220338983,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":71.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.35593220338983,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 513.842. Standardfehler: 2.023. Kosten je Aufgabe in USD: 0.448911. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 513.842. Standard error: 2.023. Cost per task in USD: 0.448911. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":66.66666666666667,"measurements":[{"observationId":"vals-terminal-bench-2:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":49.438,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":43,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 875.602. Standardfehler: 5.33. Kosten je Aufgabe in USD: 0.247234. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Top P: 1. Maximum output tokens: 8192. Latency in seconds: 875.602. Standard error: 5.33. Cost per task in USD: 0.247234. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 1. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Top P: 1. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":58.108108108108105,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":62.898,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.108108108108105,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 68.577. Standardfehler: 0.95. Kosten je Aufgabe in USD: 0.067926. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 68.577. Standard error: 0.95. Cost per task in USD: 0.067926. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.868,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 289.752. Standardfehler: 1.057. Kosten je Aufgabe in USD: 0.043918. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 289.752. Standard error: 1.057. Cost per task in USD: 0.043918. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":65.38461538461539,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.059,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.38461538461539,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 16.056. Standardfehler: 0.411. Kosten je Aufgabe in USD: 0.002311. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 16.056. Standard error: 0.411. Cost per task in USD: 0.002311. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":71.7948717948718,"measurements":[{"observationId":"vals-medqa:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":94.267,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.7948717948718,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 81.08. Standardfehler: 0.212. Kosten je Aufgabe in USD: 0.008007. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 81.08. Standard error: 0.212. Cost per task in USD: 0.008007. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":55.84415584415584,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.03,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.84415584415584,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 86.889. Standardfehler: 0.343. Kosten je Aufgabe in USD: 0.011542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 86.889. Standard error: 0.343. Cost per task in USD: 0.011542. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":22.36842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":70.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 149.298. Standardfehler: 0.9. Kosten je Aufgabe in USD: 0.021913. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 149.298. Standard error: 0.9. Cost per task in USD: 0.021913. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":27.77777777777778,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":23.359,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.77777777777778,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5-thinking. Name des Quellmodells: GLM-5. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 13455.794. Standardfehler: 4.034. Kosten je Aufgabe in USD: 40.270258. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5-thinking. Source model name: GLM-5. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 13455.794. Standard error: 4.034. Cost per task in USD: 40.270258. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":11,"totalFamilies":71,"sourceCount":2,"coverage":0.10963718820861679,"lowerBound":5.869700173096375,"upperBound":94.9059813522347,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":4,"totalFamilies":15,"sourceCount":1,"coverage":0.1777777777777778,"lowerBound":6.668093363008619,"upperBound":88.89031558523084,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":3.195488721804511,"upperBound":96.05263157894737,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":18.614718614718615,"upperBound":85.28138528138528,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":1.923076923076923,"upperBound":93.58974358974358,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":11.496647022962811,"upperBound":82.92521845153425,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":16.346153846153847,"upperBound":91.34615384615384,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":23.931623931623932,"upperBound":90.5982905982906,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-5.1","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":964.54,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":73.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 73,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 73.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":51.21951219512195,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":20.308354687857076,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.21951219512195,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 20,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 20.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":39.63414634146341,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":8.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.63414634146341,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 8,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 8.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":52.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":1116.24,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":40.254237288135606,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 40,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 40.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":56.09756097560975,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":0.85,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":51.31578947368421,"measurements":[{"observationId":"vals-gpqa:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":84.518,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.31578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 249.704. Standardfehler: 1.821. Kosten je Aufgabe in USD: 0.044897. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 249.704. Standard error: 1.821. Cost per task in USD: 0.044897. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":30.0741427247451,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 30,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 30.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":30.555555555555557,"measurements":[{"observationId":"lmarena:webdev:overall:glm-5.1","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1508.3945879043822,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":30.555555555555557,"participants":37,"methodology":{"de":"Arena-Bewertung aus 10.746 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.1. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,746 published pairwise votes.. Source model ID: lmarena/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Z.ai. Model snapshot: glm-5.1. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.746 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 10,746 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.746 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 10,746 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":36.111111111111114,"measurements":[{"observationId":"lmarena:overall:glm-5.1","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1465.3744283084607,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":36.111111111111114,"participants":37,"methodology":{"de":"Arena-Bewertung aus 62.175 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.1. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 62,175 published pairwise votes.. Source model ID: lmarena/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Z.ai. Model snapshot: glm-5.1. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 62.175 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 62,175 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 62.175 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 62,175 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":50.847457627118644,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":76.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.847457627118644,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 527.071. Standardfehler: 1.901. Kosten je Aufgabe in USD: 0.456603. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 527.071. Standard error: 1.901. Cost per task in USD: 0.456603. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":55.08710801393729,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:5aa1c578-af76-4b91-8699-cdd43582b3af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=3791:suffix=Reasoning","retrievedAt":"2026-10-19","value":2.02020202020202,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 2,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5aa1c578-af76-4b91-8699-cdd43582b3af/glm-5-1/glm-5-1. Source model name: GLM-5.1. Provider reported by source: Z AI. Model snapshot: GLM-5.1 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 2.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.1 (Reasoning) (Release glm-5-1, Zeile glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: GLM-5.1 (Reasoning) (release glm-5-1, row glm-5-1, UUID 5aa1c578-af76-4b91-8699-cdd43582b3af, reasoning=true, source reasoning-token configuration budget=3791, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":53.933,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.42857142857143,"participants":43,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 881.675. Standardfehler: 5.314. Kosten je Aufgabe in USD: 0.158286. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 881.675. Standard error: 5.314. Cost per task in USD: 0.158286. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":56.929,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.714285714285715,"participants":57,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 940.943. Standardfehler: 1.35. Kosten je Aufgabe in USD: 0.232386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 940.943. Standard error: 1.35. Cost per task in USD: 0.232386. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":43.54838709677419,"measurements":[{"observationId":"vals-code-migration:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":25.768,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.54838709677419,"participants":63,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3502.363. Standardfehler: 4.087. Kosten je Aufgabe in USD: 5.701313. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3502.363. Standard error: 4.087. Cost per task in USD: 5.701313. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":67.56756756756756,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":64.452,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.56756756756756,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Latenz in Sekunden: 68.17. Standardfehler: 0.943. Kosten je Aufgabe in USD: 0.078922. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Latency in seconds: 68.17. Standard error: 0.943. Cost per task in USD: 0.078922. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":30,"measurements":[{"observationId":"vals-finance-agent-v2:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":44.792,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 924.078. Standardfehler: 0.606. Kosten je Aufgabe in USD: 0.618185. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 924.078. Standard error: 0.606. Cost per task in USD: 0.618185. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":40.25974025974026,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.38,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.25974025974026,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 226.424. Standardfehler: 1.061. Kosten je Aufgabe in USD: 0.045983. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 226.424. Standard error: 1.061. Cost per task in USD: 0.045983. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 498.074. Standardfehler: 0. Kosten je Aufgabe in USD: 0.555636. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 498.074. Standard error: 0. Cost per task in USD: 0.555636. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":70.51282051282051,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.394,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.51282051282051,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Latenz in Sekunden: 13.749. Standardfehler: 0.429. Kosten je Aufgabe in USD: 0.002183. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Latency in seconds: 13.749. Standard error: 0.429. Cost per task in USD: 0.002183. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":38.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":27.885,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 325.824. Standardfehler: 3.117. Kosten je Aufgabe in USD: 0.385737. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 325.824. Standard error: 3.117. Cost per task in USD: 0.385737. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":43.28358208955224,"measurements":[{"observationId":"vals-medcode:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":41.604,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.28358208955224,"participants":68,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 77.578. Standardfehler: 2.124. Kosten je Aufgabe in USD: 0.024196. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 77.578. Standard error: 2.124. Cost per task in USD: 0.024196. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":16.176470588235293,"measurements":[{"observationId":"vals-medscribe:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":72.27,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.176470588235293,"participants":69,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 95.701. Standardfehler: 2.064. Kosten je Aufgabe in USD: 0.023717. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 95.701. Standard error: 2.064. Cost per task in USD: 0.023717. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":68.83116883116882,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.9,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.83116883116882,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 69.318. Standardfehler: 0.334. Kosten je Aufgabe in USD: 0.012383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 69.318. Standard error: 0.334. Cost per task in USD: 0.012383. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3931.735. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3931.735. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":42.10526315789474,"measurements":[{"observationId":"vals-snap:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":61.84,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.10526315789474,"participants":39,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 580.381. Standardfehler: 1.264. Kosten je Aufgabe in USD: 0.340666. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 580.381. Standard error: 1.264. Cost per task in USD: 0.340666. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":31.57894736842105,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.194,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.57894736842105,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 84.072. Standardfehler: 0.895. Kosten je Aufgabe in USD: 0.016958. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Latency in seconds: 84.072. Standard error: 0.895. Cost per task in USD: 0.016958. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":31.944444444444443,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-5.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":31.456,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.944444444444443,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.1. Name des Quellmodells: GLM-5.1. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2014.732. Standardfehler: 4.553. Kosten je Aufgabe in USD: 2.894911. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.1. Source model name: GLM-5.1. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2014.732. Standard error: 4.553. Cost per task in USD: 2.894911. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":27,"totalFamilies":71,"sourceCount":3,"coverage":0.31400226757369615,"lowerBound":13.547459521476725,"upperBound":82.1472327641071,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.42222222222222233,"lowerBound":16.14183836222593,"upperBound":73.9196161400037,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":17.776911791674305,"upperBound":74.91976893453145,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":62.46436490338929,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":41.64290993559286,"upperBound":74.9762432689262,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":17.682926829268293,"upperBound":84.34959349593497,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":7.222222222222223,"upperBound":87.22222222222223,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":18.44950213371266,"upperBound":75.5923592765698,"provisional":true,"configurationsDiffer":true},"legal":{"score":39.8339638503573,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":29.875472887767973,"upperBound":54.87547288776797,"provisional":true,"configurationsDiffer":true},"health":{"score":29.73002633889377,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":19.820017559262514,"upperBound":53.15335089259585,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":21.05263157894737,"upperBound":71.05263157894737,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-5.2","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"research:glm-5.2:31","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.8638888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":9.090909090909092,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: XjFqou6MyT2XLpkUsCrgxM. Startzeit: 2026-06-25T00:15:08.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.8638888888888889 und stderr=0.04219779330510574. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: XjFqou6MyT2XLpkUsCrgxM. Started at 2026-06-25T00:15:08.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.8638888888888889 and stderr=0.04219779330510574. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1228.86,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":53.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":78.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 78,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 78.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":33.7020648967552,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.92307692307692,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 33,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 33.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":28.398977210054348,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 28,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 28.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":10.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 10,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 10.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1372.01,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":81.15942028985508,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":73.3333333333333,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.15942028985508,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 73,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 73.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":65.71428571428571,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":42.6553672316384,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.71428571428571,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 42,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 42.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":4.433333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":9.523809523809524,"measurements":[{"observationId":"research:glm-5.2:27","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.22,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":9.523809523809524,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: Hui8iZgjwKPxz7DkRU8sJX. Startzeit: 2026-10-14T05:10:27.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.22 und stderr=0.041633319989322654. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: Hui8iZgjwKPxz7DkRU8sJX. Started at 2026-10-14T05:10:27.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.22 and stderr=0.041633319989322654. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"research:glm-5.2:29","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.342,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":11.11111111111111,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: WgjFZzojpiv54XqUhvfuUG. Startzeit: 2026-08-27T19:36:05.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.342 und stderr=0.015008706182121802. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: WgjFZzojpiv54XqUhvfuUG. Started at 2026-08-27T19:36:05.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.342 and stderr=0.015008706182121802. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":33.58585858585859,"measurements":[{"observationId":"research:glm-5.2:22","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.2926829268292683,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":44.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: JebA8dYLVov6H49ZQSLXh5. Startzeit: 2026-06-19T16:13:05.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.2926829268292683 und stderr=0.07194088392074452. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: JebA8dYLVov6H49ZQSLXh5. Started at 2026-06-19T16:13:05.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.2926829268292683 and stderr=0.07194088392074452. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:glm-5.2:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.592057761732852,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":22.727272727272727,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: TTuCPacMDpNaCSs3Hpbdjp. Startzeit: 2026-06-19T16:13:05.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.592057761732852 und stderr=0.029581952519606228. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: TTuCPacMDpNaCSs3Hpbdjp. Started at 2026-06-19T16:13:05.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.592057761732852 and stderr=0.029581952519606228. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":49.06015037593985,"measurements":[{"observationId":"research:glm-5.2:24","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9185606060606061,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":42.857142857142854,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.2_max. Epoch-Modellbezeichnung: GLM-5.2. Anzeigenamenfeld: GLM-5.2. Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: c5kpB4axzmygRywWqBKFmv. Startzeit: 2026-06-24T20:55:39.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9185606060606061 und stderr=0.016123207533687733. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.2_max; Epoch model label: GLM-5.2; Display label field: GLM-5.2; Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: c5kpB4axzmygRywWqBKFmv. Started at 2026-06-24T20:55:39.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9185606060606061 and stderr=0.016123207533687733. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":85.606,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 224.664. Standardfehler: 1.774. Kosten je Aufgabe in USD: 0.05711. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 224.664. Standard error: 1.774. Cost per task in USD: 0.05711. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":41.1492122335496,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":52.77777777777778,"measurements":[{"observationId":"lmarena:webdev:overall:glm-5.2-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1604.1131408266187,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":52.77777777777778,"participants":37,"methodology":{"de":"Arena-Bewertung aus 16.093 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.2-max. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.2-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 16,093 published pairwise votes.. Source model ID: lmarena/glm-5.2-max. Source model name: GLM-5.2. Provider reported by source: Z.ai. Model snapshot: glm-5.2-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 16.093 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 16,093 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 16.093 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 16,093 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"lmarena:overall:glm-5.2-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1475.4702477730777,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":55.55555555555556,"participants":37,"methodology":{"de":"Arena-Bewertung aus 49.127 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.2-max. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.2-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 49,127 published pairwise votes.. Source model ID: lmarena/glm-5.2-max. Source model name: GLM-5.2. Provider reported by source: Z.ai. Model snapshot: glm-5.2-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 49.127 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 49,127 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 49.127 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 49,127 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":77.96610169491525,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":82.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.96610169491525,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 669.667. Standardfehler: 1.689. Kosten je Aufgabe in USD: 0.710002. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 669.667. Standard error: 1.689. Cost per task in USD: 0.710002. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":50.212325783972126,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:f7a4ea75-e548-4069-80d4-9be8bc7c009b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.2 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7a4ea75-e548-4069-80d4-9be8bc7c009b/glm-5-2/glm-5-2. Source model name: GLM-5.2. Provider reported by source: Z AI. Model snapshot: GLM-5.2 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.2 (Max) (Release glm-5-2, Zeile glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.2 (Max) (release glm-5-2, row glm-5-2, UUID f7a4ea75-e548-4069-80d4-9be8bc7c009b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":67.79,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.67857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 48000. Latenz in Sekunden: 820.506. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.425184. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 48000. Latency in seconds: 820.506. Standard error: 0.991. Cost per task in USD: 0.425184. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 48000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 48000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 48000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 48000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":61.29032258064516,"measurements":[{"observationId":"vals-code-migration:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":37.87,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.29032258064516,"participants":63,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 6856.367. Standardfehler: 4.143. Kosten je Aufgabe in USD: 12.635818. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 6856.367. Standard error: 4.143. Cost per task in USD: 12.635818. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":79.72972972972973,"measurements":[{"observationId":"vals-corp-fin-v2:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.123,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.72972972972973,"participants":75,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Latenz in Sekunden: 24.586. Standardfehler: 0.932. Kosten je Aufgabe in USD: 0.121857. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Latency in seconds: 24.586. Standard error: 0.932. Cost per task in USD: 0.121857. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":55.932203389830505,"measurements":[{"observationId":"vals-emb:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":61.532,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.932203389830505,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Latenz in Sekunden: 3532.407. Standardfehler: 2.879. Kosten je Aufgabe in USD: 4.465213. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Latency in seconds: 3532.407. Standard error: 2.879. Cost per task in USD: 4.465213. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":49.699,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Latenz in Sekunden: 473.158. Standardfehler: 0.878. Kosten je Aufgabe in USD: 0.713981. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Latency in seconds: 473.158. Standard error: 0.878. Cost per task in USD: 0.713981. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":19.48051948051948,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":69.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.48051948051948,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 532.222. Standardfehler: 1.179. Kosten je Aufgabe in USD: 0.129308. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 532.222. Standard error: 1.179. Cost per task in USD: 0.129308. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":75.40983606557377,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":7.083,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.40983606557377,"participants":62,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1250.956. Standardfehler: 1.998. Kosten je Aufgabe in USD: 2.057915. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1250.956. Standard error: 1.998. Cost per task in USD: 2.057915. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":67.3076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.073,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.3076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Latenz in Sekunden: 5.766. Standardfehler: 0.452. Kosten je Aufgabe in USD: 0.001596. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Latency in seconds: 5.766. Standard error: 0.452. Cost per task in USD: 0.001596. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":48.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":31.25,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1023.468. Standardfehler: 3.222. Kosten je Aufgabe in USD: 0.889209. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1023.468. Standard error: 3.222. Cost per task in USD: 0.889209. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":35.82089552238806,"measurements":[{"observationId":"vals-medcode:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.771,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.82089552238806,"participants":68,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 95.265. Standardfehler: 2.166. Kosten je Aufgabe in USD: 0.04501. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 95.265. Standard error: 2.166. Cost per task in USD: 0.04501. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":52.94117647058823,"measurements":[{"observationId":"vals-medscribe:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.534,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.94117647058823,"participants":69,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 138.372. Standardfehler: 2.002. Kosten je Aufgabe in USD: 0.044912. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 138.372. Standard error: 2.002. Cost per task in USD: 0.044912. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":66.23376623376623,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.23376623376623,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 51.64. Standardfehler: 0.334. Kosten je Aufgabe in USD: 0.013838. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 51.64. Standard error: 0.334. Cost per task in USD: 0.013838. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 64000. Latenz in Sekunden: 7588.932. Standardfehler: 0.5. Kosten je Aufgabe in USD: 12.57696. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 64000. Latency in seconds: 7588.932. Standard error: 0.5. Cost per task in USD: 12.57696. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 64000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 64000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 64000","en":"Reasoning effort: max. Temperature: 1. Top P: 1. Maximum output tokens: 64000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":27.586206896551722,"measurements":[{"observationId":"vals-skillsbench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":45.077,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.586206896551722,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1191.665. Standardfehler: 4.392. Kosten je Aufgabe in USD: 0.811017. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1191.665. Standard error: 4.392. Cost per task in USD: 0.811017. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":4.166666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.166666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 8960.277. Standardfehler: 0. Kosten je Aufgabe in USD: 26.575264. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 8960.277. Standard error: 0. Cost per task in USD: 26.575264. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":55.26315789473684,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.344,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 64.942. Standardfehler: 0.865. Kosten je Aufgabe in USD: 0.015431. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Latency in seconds: 64.942. Standard error: 0.865. Cost per task in USD: 0.015431. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":54.166666666666664,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-5.2","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":63.961,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.166666666666664,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.2. Name des Quellmodells: GLM-5.2. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3748.167. Standardfehler: 4.795. Kosten je Aufgabe in USD: 8.488514. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.2. Source model name: GLM-5.2. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3748.167. Standard error: 4.795. Cost per task in USD: 8.488514. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":35,"totalFamilies":71,"sourceCount":4,"coverage":0.3958616780045353,"lowerBound":19.53401047257578,"upperBound":79.94784267212225,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.41111111111111126,"lowerBound":21.35848685611342,"upperBound":80.2473757450023,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":42.55795314845379,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":27.35868416686315,"upperBound":63.07296988114887,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":8.535353535353536,"upperBound":68.53535353535354,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":46.9198371637396,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":46.9198371637396,"upperBound":46.9198371637396,"provisional":true,"configurationsDiffer":true},"agentic":{"score":59.671407930449135,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":29.835703965224567,"upperBound":79.83570396522457,"provisional":true,"configurationsDiffer":true},"general":{"score":46.96054750402576,"coveredFamilies":3,"totalFamilies":5,"sourceCount":3,"coverage":0.6000000000000001,"lowerBound":28.17632850241546,"upperBound":68.17632850241546,"provisional":true,"configurationsDiffer":true},"finance":{"score":59.39793942024094,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":33.94167966870911,"upperBound":76.79882252585196,"provisional":true,"configurationsDiffer":true},"legal":{"score":63.68362056886647,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":47.762715426649855,"upperBound":72.76271542664986,"provisional":true,"configurationsDiffer":true},"health":{"score":44.381035996488144,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":29.587357330992095,"upperBound":62.920690664325434,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-5.3","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"research:glm-5.3:22","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9111111111111111,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":18.181818181818183,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: goZR5yGxZic9TZ23ezCL3w. Startzeit: 2026-08-24T19:44:18.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9111111111111111 und stderr=0.03454754040013868. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: goZR5yGxZic9TZ23ezCL3w. Started at 2026-08-24T19:44:18.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9111111111111111 and stderr=0.03454754040013868. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":43.421052631578945,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":38.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.421052631578945,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 38,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 38.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":86.58536585365853,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1506.86,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":79.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 79,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 79.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":62.202864996264196,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 62,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 62.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":40.0752,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.333333333333336,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 40,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 40.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":50.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":11.200000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 11,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 11.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1650.31,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":46.0828625235405,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.42857142857143,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 46,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 46.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":14.3,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"research:glm-5.3:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.37,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":33.333333333333336,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: kG3hbd8qLEd6fLAJHyQTiN. Startzeit: 2026-10-13T21:22:18.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.37 und stderr=0.048523658709390974. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: kG3hbd8qLEd6fLAJHyQTiN. Started at 2026-10-13T21:22:18.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.37 and stderr=0.048523658709390974. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"research:glm-5.3:21","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.41,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":16.666666666666668,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: TY8R8FV7zCwByEipkqyg2P. Startzeit: 2026-08-28T17:00:06.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.41 und stderr=0.015560917136921659. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: TY8R8FV7zCwByEipkqyg2P. Started at 2026-08-28T17:00:06.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.41 and stderr=0.015560917136921659. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":44.94949494949495,"measurements":[{"observationId":"research:glm-5.3:18","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.2926829268292683,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":44.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: 4S3ZrJDkwh4gmGrmJQdvzR. Startzeit: 2026-08-25T14:20:20.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.2926829268292683 und stderr=0.07194088392074452. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: 4S3ZrJDkwh4gmGrmJQdvzR. Started at 2026-08-25T14:20:20.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.2926829268292683 and stderr=0.07194088392074452. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:glm-5.3:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.6877192982456141,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":45.45454545454545,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: iXvqUzwxAiA3Rg3dwnutFs. Startzeit: 2026-08-25T14:20:20.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.6877192982456141 und stderr=0.027499133473298815. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: iXvqUzwxAiA3Rg3dwnutFs. Started at 2026-08-25T14:20:20.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.6877192982456141 and stderr=0.027499133473298815. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":51.2844611528822,"measurements":[{"observationId":"research:glm-5.3:19","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9090909090909091,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":38.095238095238095,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3_max. Epoch-Modellbezeichnung: GLM-5.3. Anzeigenamenfeld: GLM-5.3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: dLDBahYBTGRNjbsXA5pyr3. Startzeit: 2026-08-24T19:44:18.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9090909090909091 und stderr=0.015851578202142853. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3_max; Epoch model label: GLM-5.3; Display label field: GLM-5.3 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: dLDBahYBTGRNjbsXA5pyr3. Started at 2026-08-24T19:44:18.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9090909090909091 and stderr=0.015851578202142853. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":88.132,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.47368421052632,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 210.7. Standardfehler: 1.851. Kosten je Aufgabe in USD: 0.055878. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 210.7. Standard error: 1.851. Cost per task in USD: 0.055878. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":42.2613531047266,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 42,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 42.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":63.888888888888886,"measurements":[{"observationId":"lmarena:webdev:overall:glm-5.3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1621.2027904936976,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":63.888888888888886,"participants":37,"methodology":{"de":"Arena-Bewertung aus 9.125 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.3-max. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 9,125 published pairwise votes.. Source model ID: lmarena/glm-5.3-max. Source model name: GLM-5.3. Provider reported by source: Z.ai. Model snapshot: glm-5.3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 9.125 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 9,125 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 9.125 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 9,125 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":63.888888888888886,"measurements":[{"observationId":"lmarena:overall:glm-5.3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1478.5284927987004,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":63.888888888888886,"participants":37,"methodology":{"de":"Arena-Bewertung aus 21.797 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.3-max. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 21,797 published pairwise votes.. Source model ID: lmarena/glm-5.3-max. Source model name: GLM-5.3. Provider reported by source: Z.ai. Model snapshot: glm-5.3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 21.797 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 21,797 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 21.797 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 21,797 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":94.0677966101695,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":95.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.0677966101695,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 840.831. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.338311. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 840.831. Standard error: 0.938. Cost per task in USD: 0.338311. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":81.25835766079669,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":41.9191919191919,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":71.536,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.92857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 786.604. Standardfehler: 1.982. Kosten je Aufgabe in USD: 0.30672. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 786.604. Standard error: 1.982. Cost per task in USD: 0.30672. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":38.889,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.97297297297297,"participants":38,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4952.947. Standardfehler: 1.821. Kosten je Aufgabe in USD: 9.37436. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 4952.947. Standard error: 1.821. Cost per task in USD: 9.37436. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":45.44179523141655,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:cd684ea4-b475-4269-b001-d469d06d8a7a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":9.52380952380952,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.17391304347826,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 9,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/cd684ea4-b475-4269-b001-d469d06d8a7a/glm-5-3/glm-5-3. Source model name: GLM-5.3. Provider reported by source: Z AI. Model snapshot: GLM-5.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 9.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3 (Max) (Release glm-5-3, Zeile glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GLM-5.3 (Max) (release glm-5-3, row glm-5-3, UUID cd684ea4-b475-4269-b001-d469d06d8a7a, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":5.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.70967741935484,"participants":32,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 9898.858. Standardfehler: 2.794. Kosten je Aufgabe in USD: 15.627955. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 9898.858. Standard error: 2.794. Cost per task in USD: 15.627955. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":67.74193548387096,"measurements":[{"observationId":"vals-code-migration:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.74193548387096,"participants":63,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 13633.637. Standardfehler: 4.289. Kosten je Aufgabe in USD: 24.909755. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 13633.637. Standard error: 4.289. Cost per task in USD: 24.909755. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":71.21212121212122,"measurements":[{"observationId":"vals-cyberbench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":72.084,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.21212121212122,"participants":34,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1818.476. Standardfehler: 5.408. Kosten je Aufgabe in USD: 2.688034. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1818.476. Standard error: 5.408. Cost per task in USD: 2.688034. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":42.3728813559322,"measurements":[{"observationId":"vals-emb:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":56.344,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.3728813559322,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2172.231. Standardfehler: 3.319. Kosten je Aufgabe in USD: 3.787827. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2172.231. Standard error: 3.319. Cost per task in USD: 3.787827. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":78.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":55.84,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 949.539. Standardfehler: 2.066. Kosten je Aufgabe in USD: 1.072768. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 949.539. Standard error: 2.066. Cost per task in USD: 1.072768. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-ioi:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":68.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":34,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6374.465. Standardfehler: 7.476. Kosten je Aufgabe in USD: 7.67087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 6374.465. Standard error: 7.476. Cost per task in USD: 7.67087. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":80.535,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 246.75. Standardfehler: 1.065. Kosten je Aufgabe in USD: 0.069397. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 246.75. Standard error: 1.065. Cost per task in USD: 0.069397. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":77.04918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":8.333,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.04918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2558.175. Standardfehler: 1.998. Kosten je Aufgabe in USD: 4.302903. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2558.175. Standard error: 1.998. Cost per task in USD: 4.302903. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":74.35897435897436,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.837,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.35897435897436,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.484. Standardfehler: 0.4. Kosten je Aufgabe in USD: 0.006513. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 22.484. Standard error: 0.4. Cost per task in USD: 0.006513. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":90,"measurements":[{"observationId":"vals-legal-research:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":49.038,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3110.171. Standardfehler: 3.475. Kosten je Aufgabe in USD: 2.241388. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3110.171. Standard error: 3.475. Cost per task in USD: 2.241388. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":46.26865671641791,"measurements":[{"observationId":"vals-medcode:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":42.864,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.26865671641791,"participants":68,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 169.01. Standardfehler: 2.111. Kosten je Aufgabe in USD: 0.081905. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 169.01. Standard error: 2.111. Cost per task in USD: 0.081905. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":88.23529411764706,"measurements":[{"observationId":"vals-medscribe:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":88.81,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.23529411764706,"participants":69,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 121.931. Standardfehler: 1.999. Kosten je Aufgabe in USD: 0.057236. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 121.931. Standard error: 1.999. Cost per task in USD: 0.057236. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":67.53246753246754,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.77,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.53246753246754,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 61.783. Standardfehler: 0.335. Kosten je Aufgabe in USD: 0.016639. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 61.783. Standard error: 0.335. Cost per task in USD: 0.016639. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":25,"measurements":[{"observationId":"vals-mysterymechanism:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":22.973,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":21,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1841.98. Standardfehler: 2.83. Kosten je Aufgabe in USD: 1.531337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1841.98. Standard error: 2.83. Cost per task in USD: 1.531337. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":78.57142857142857,"measurements":[{"observationId":"vals-programbench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":1.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.57142857142857,"participants":50,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15354.08. Standardfehler: 0.862. Kosten je Aufgabe in USD: 21.957364. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 15354.08. Standard error: 0.862. Cost per task in USD: 21.957364. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":43.58974358974359,"measurements":[{"observationId":"vals-proof-bench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":49,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.58974358974359,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2542.672. Standardfehler: 5.024. Kosten je Aufgabe in USD: 2.081538. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2542.672. Standard error: 5.024. Cost per task in USD: 2.081538. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":34.48275862068966,"measurements":[{"observationId":"vals-skillsbench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":47.509,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.48275862068966,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 787.365. Standardfehler: 4.522. Kosten je Aufgabe in USD: 0.71406. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 787.365. Standard error: 4.522. Cost per task in USD: 0.71406. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":84.21052631578948,"measurements":[{"observationId":"vals-snap:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":68.539,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.21052631578948,"participants":39,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2150.623. Standardfehler: 1.208. Kosten je Aufgabe in USD: 0.946832. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2150.623. Standard error: 1.208. Cost per task in USD: 0.946832. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":54.166666666666664,"measurements":[{"observationId":"vals-sre-bench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":9.16,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.166666666666664,"participants":25,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 10089.867. Standardfehler: 1.786. Kosten je Aufgabe in USD: 27.465302. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 10089.867. Standard error: 1.786. Cost per task in USD: 27.465302. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"vals-tax-agent-bench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":39.698,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":56,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3052.932. Standardfehler: 3.52. Kosten je Aufgabe in USD: 1.80097. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3052.932. Standard error: 3.52. Cost per task in USD: 1.80097. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":46.05263157894737,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.363,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 98.323. Standardfehler: 0.879. Kosten je Aufgabe in USD: 0.032. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 98.323. Standard error: 0.879. Cost per task in USD: 0.032. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":57.142857142857146,"measurements":[{"observationId":"vals-vcb-1-100:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":19.994,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.142857142857146,"participants":22,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 5169.473. Standardfehler: 3.811. Kosten je Aufgabe in USD: 8.236953. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 5169.473. Standard error: 3.811. Cost per task in USD: 8.236953. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":70.83333333333333,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-5.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":78.125,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.83333333333333,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3. Name des Quellmodells: GLM-5.3. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3847.197. Standardfehler: 3.94. Kosten je Aufgabe in USD: 12.450185. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3. Source model name: GLM-5.3. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3847.197. Standard error: 3.94. Cost per task in USD: 12.450185. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":62.287114438688185,"coveredFamilies":42,"totalFamilies":71,"sourceCount":4,"coverage":0.5371315192743764,"lowerBound":33.456372409669534,"upperBound":79.74322048223189,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":61.425781100436836,"coveredFamilies":11,"totalFamilies":15,"sourceCount":3,"coverage":0.688888888888889,"lowerBound":42.315538091412044,"upperBound":73.42664920252315,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":51.866284953576354,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":33.342611755870514,"upperBound":69.05689747015623,"provisional":true,"configurationsDiffer":true},"math":{"score":35.57368557368557,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":21.344211344211345,"upperBound":61.34421134421134,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":52.45662196881709,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":52.45662196881709,"upperBound":52.45662196881709,"provisional":true,"configurationsDiffer":true},"agentic":{"score":70.01068166631887,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":35.00534083315944,"upperBound":85.00534083315944,"provisional":true,"configurationsDiffer":true},"general":{"score":47.68518518518518,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":28.61111111111111,"upperBound":68.6111111111111,"provisional":true,"configurationsDiffer":true},"finance":{"score":64.41698429432594,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":36.809705311043395,"upperBound":79.66684816818625,"provisional":true,"configurationsDiffer":true},"legal":{"score":80.46938489561441,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":60.3520386717108,"upperBound":85.3520386717108,"provisional":true,"configurationsDiffer":true},"health":{"score":67.25197541703248,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":44.834650278021655,"upperBound":78.167983611355,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":71.21212121212122,"upperBound":71.21212121212122,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":42.10526315789474,"upperBound":92.10526315789474,"provisional":true,"configurationsDiffer":true}}},{"modelId":"glm-5.3-flash","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":22.727272727272727,"measurements":[{"observationId":"research:glm-5.3-flash:21","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9388888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":22.727272727272727,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3-flash_max. Epoch-Modellbezeichnung: GLM-5.3-Flash. Anzeigenamenfeld: GLM-5.3-Flash (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: NCikvVWc5SEyMEbCqvVNb2. Startzeit: 2026-08-26T22:26:55.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9388888888888889 und stderr=0.021622267380853318. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3-flash_max; Epoch model label: GLM-5.3-Flash; Display label field: GLM-5.3-Flash (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: NCikvVWc5SEyMEbCqvVNb2. Started at 2026-08-26T22:26:55.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9388888888888889 and stderr=0.021622267380853318. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":28.94736842105263,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.94736842105263,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 25,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 25.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1452.63,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":80,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 80,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 80.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":60.36862782167782,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 60,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 60.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":40.9957333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.666666666666664,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Angezeigter Wert: 41,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Score display: 41.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":15.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 15,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 15.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1642.18,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":94.28571428571429,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":51.24293785310739,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":94.28571428571429,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 51,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 51.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":7.466666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":4.761904761904762,"measurements":[{"observationId":"research:glm-5.3-flash:20","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.21,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":4.761904761904762,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3-flash_max. Epoch-Modellbezeichnung: GLM-5.3-Flash. Anzeigenamenfeld: GLM-5.3-Flash (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: fWNxmiSUC7BX8whqvzssY8. Startzeit: 2026-10-13T15:54:34.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.21 und stderr=0.040936018074033236. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3-flash_max; Epoch model label: GLM-5.3-Flash; Display label field: GLM-5.3-Flash (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: fWNxmiSUC7BX8whqvzssY8. Started at 2026-10-13T15:54:34.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.21 and stderr=0.040936018074033236. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":8.712121212121213,"measurements":[{"observationId":"research:glm-5.3-flash:18","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.17073170731707318,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":8.333333333333334,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3-flash_max. Epoch-Modellbezeichnung: GLM-5.3-Flash. Anzeigenamenfeld: GLM-5.3-Flash (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: iCysWvFJevNyw2dbYosU6J. Startzeit: 2026-08-27T13:29:52.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.17073170731707318 und stderr=0.05949419959829495. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3-flash_max; Epoch model label: GLM-5.3-Flash; Display label field: GLM-5.3-Flash (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: iCysWvFJevNyw2dbYosU6J. Started at 2026-08-27T13:29:52.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.17073170731707318 and stderr=0.05949419959829495. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:glm-5.3-flash:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.5578947368421052,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":9.090909090909092,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3-flash_max. Epoch-Modellbezeichnung: GLM-5.3-Flash. Anzeigenamenfeld: GLM-5.3-Flash (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: YC2u9wykbYQg4gdkf76j26. Startzeit: 2026-08-27T13:29:52.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.5578947368421052 und stderr=0.029469977528223976. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3-flash_max; Epoch model label: GLM-5.3-Flash; Display label field: GLM-5.3-Flash (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: YC2u9wykbYQg4gdkf76j26. Started at 2026-08-27T13:29:52.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.5578947368421052 and stderr=0.029469977528223976. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":43.890977443609025,"measurements":[{"observationId":"research:glm-5.3-flash:19","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9015151515151515,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":28.571428571428573,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: glm-5.3-flash_max. Epoch-Modellbezeichnung: GLM-5.3-Flash. Anzeigenamenfeld: GLM-5.3-Flash (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: Jvt5BNr4ERbE9QGgMcEbmT. Startzeit: 2026-08-26T22:26:55.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9015151515151515 und stderr=0.017183231147411448. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: glm-5.3-flash_max; Epoch model label: GLM-5.3-Flash; Display label field: GLM-5.3-Flash (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: Jvt5BNr4ERbE9QGgMcEbmT. Started at 2026-08-26T22:26:55.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9015151515151515 and stderr=0.017183231147411448. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":86.364,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.21052631578947,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 131.004. Standardfehler: 2.126. Kosten je Aufgabe in USD: 0.001626. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 131.004. Standard error: 2.126. Cost per task in USD: 0.001626. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":39.8517145505097,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 39,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 39.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"lmarena:webdev:overall:glm-5.3-flash","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1606.0511899791893,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":55.55555555555556,"participants":37,"methodology":{"de":"Arena-Bewertung aus 16.470 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.3-flash. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 16,470 published pairwise votes.. Source model ID: lmarena/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z.ai. Model snapshot: glm-5.3-flash. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 16.470 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 16,470 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 16.470 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 16,470 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":47.22222222222222,"measurements":[{"observationId":"lmarena:overall:glm-5.3-flash","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1474.2282356112491,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":47.22222222222222,"participants":37,"methodology":{"de":"Arena-Bewertung aus 31.379 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.3-flash. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 31,379 published pairwise votes.. Source model ID: lmarena/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z.ai. Model snapshot: glm-5.3-flash. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 31.379 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 31,379 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 31.379 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 31,379 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"lmarena:vision_style_control:overall:glm-5.3-flash","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1273.3307750524925,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":53.333333333333336,"participants":31,"methodology":{"de":"Arena-Bewertung aus 11.157 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z.ai. Modellstand: glm-5.3-flash. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,157 published pairwise votes.. Source model ID: lmarena/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z.ai. Model snapshot: glm-5.3-flash. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.157 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 11,157 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.157 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 11,157 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":63.829787234042556,"measurements":[{"observationId":"vals-mmmu:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":86.012,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.829787234042556,"participants":48,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 28.218. Standardfehler: 0.834. Kosten je Aufgabe in USD: 0.000962. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 28.218. Standard error: 0.834. Cost per task in USD: 0.000962. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":86.44067796610169,"measurements":[{"observationId":"vals-swebench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":92,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.44067796610169,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3817.379. Standardfehler: 1.214. Kosten je Aufgabe in USD: 0.017996. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3817.379. Standard error: 1.214. Cost per task in USD: 0.017996. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":73.65288633581316,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":32.8282828282828,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.7560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 32,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 32.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":62.921,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.42857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 965.737. Standardfehler: 2.247. Kosten je Aufgabe in USD: 0.028403. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 965.737. Standard error: 2.247. Cost per task in USD: 0.028403. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":37.374,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.56756756756756,"participants":38,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3681.073. Standardfehler: 2.672. Kosten je Aufgabe in USD: 1.219907. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3681.073. Standard error: 2.672. Cost per task in USD: 1.219907. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":23.807854137447407,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:19496b81-9f41-4214-a77a-1df803b3c5ae","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":4.76190476190476,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.26086956521739,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Z AI. Modellstand: GLM-5.3-Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 4,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/19496b81-9f41-4214-a77a-1df803b3c5ae/glm-5-3-flash/glm-5-3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Z AI. Model snapshot: GLM-5.3-Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 4.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GLM-5.3-Flash (Release glm-5-3-flash, Zeile glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: GLM-5.3-Flash (release glm-5-3-flash, row glm-5-3-flash, UUID 19496b81-9f41-4214-a77a-1df803b3c5ae, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":2.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.35483870967742,"participants":32,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12461.546. Standardfehler: 2.006. Kosten je Aufgabe in USD: 0.524048. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 12461.546. Standard error: 2.006. Cost per task in USD: 0.524048. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":37.096774193548384,"measurements":[{"observationId":"vals-code-migration:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":20.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.096774193548384,"participants":63,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12331.871. Standardfehler: 4.207. Kosten je Aufgabe in USD: 3.128822. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 12331.871. Standard error: 4.207. Cost per task in USD: 3.128822. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-cyberbench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":67.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":34,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2148.165. Standardfehler: 5.566. Kosten je Aufgabe in USD: 0.142012. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2148.165. Standard error: 5.566. Cost per task in USD: 0.142012. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":38.983050847457626,"measurements":[{"observationId":"vals-emb:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":55.932,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.983050847457626,"participants":60,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4495.202. Standardfehler: 3.561. Kosten je Aufgabe in USD: 0.713033. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 4495.202. Standard error: 3.561. Cost per task in USD: 0.713033. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":85,"measurements":[{"observationId":"vals-finance-agent-v2:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":57.85,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 890.519. Standardfehler: 2.061. Kosten je Aufgabe in USD: 0.047625. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 890.519. Standard error: 2.061. Cost per task in USD: 0.047625. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":42.42424242424242,"measurements":[{"observationId":"vals-ioi:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":52.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.42424242424242,"participants":34,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5548.524. Standardfehler: 2.619. Kosten je Aufgabe in USD: 0.429263. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 5548.524. Standard error: 2.619. Cost per task in USD: 0.429263. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":35.064935064935064,"measurements":[{"observationId":"vals-lcb:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":80.51,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.064935064935064,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 301.901. Standardfehler: 1.051. Kosten je Aufgabe in USD: 0.007053. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 301.901. Standard error: 1.051. Cost per task in USD: 0.007053. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":70.49180327868852,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":6.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.49180327868852,"participants":62,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3136.646. Standardfehler: 1.646. Kosten je Aufgabe in USD: 0.570286. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3136.646. Standard error: 1.646. Cost per task in USD: 0.570286. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"vals-legal-bench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.928,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":79,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.521. Standardfehler: 0.427. Kosten je Aufgabe in USD: 0.000278. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 12.521. Standard error: 0.427. Cost per task in USD: 0.000278. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":78.33333333333333,"measurements":[{"observationId":"vals-legal-research:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":45.192,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3067.486. Standardfehler: 3.459. Kosten je Aufgabe in USD: 0.120196. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3067.486. Standard error: 3.459. Cost per task in USD: 0.120196. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":89.70588235294117,"measurements":[{"observationId":"vals-medscribe:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":88.936,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.70588235294117,"participants":69,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 86.98. Standardfehler: 1.907. Kosten je Aufgabe in USD: 0.002235. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 86.98. Standard error: 1.907. Cost per task in USD: 0.002235. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":58.44155844155844,"measurements":[{"observationId":"vals-mmlu-pro:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.059,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.44155844155844,"participants":78,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 44.762. Standardfehler: 0.341. Kosten je Aufgabe in USD: 0.001058. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 44.762. Standard error: 0.341. Cost per task in USD: 0.001058. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 19801.258. Standardfehler: 0. Kosten je Aufgabe in USD: 0.79862. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 19801.258. Standard error: 0. Cost per task in USD: 0.79862. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":25.641025641025642,"measurements":[{"observationId":"vals-proof-bench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":21,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.641025641025642,"participants":40,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4274.803. Standardfehler: 4.094. Kosten je Aufgabe in USD: 0.224234. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 4274.803. Standard error: 4.094. Cost per task in USD: 0.224234. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":17.24137931034483,"measurements":[{"observationId":"vals-skillsbench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":40.159,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.24137931034483,"participants":30,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 728.997. Standardfehler: 4.432. Kosten je Aufgabe in USD: 0.039052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 728.997. Standard error: 4.432. Cost per task in USD: 0.039052. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":20.833333333333332,"measurements":[{"observationId":"vals-sre-bench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0.763,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.833333333333332,"participants":25,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 11693.322. Standardfehler: 0.539. Kosten je Aufgabe in USD: 2.290191. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 11693.322. Standard error: 0.539. Cost per task in USD: 2.290191. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"vals-tax-agent-bench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":31.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.72727272727273,"participants":56,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1297.006. Standardfehler: 3.35. Kosten je Aufgabe in USD: 0.138062. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1297.006. Standard error: 3.35. Cost per task in USD: 0.138062. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":89.47368421052632,"measurements":[{"observationId":"vals-tax-eval-v2:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.593,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.47368421052632,"participants":77,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 53.06. Standardfehler: 0.836. Kosten je Aufgabe in USD: 0.001505. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 53.06. Standard error: 0.836. Cost per task in USD: 0.001505. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":38.095238095238095,"measurements":[{"observationId":"vals-vcb-1-100:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":16.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.095238095238095,"participants":22,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2343.096. Standardfehler: 3.279. Kosten je Aufgabe in USD: 0.462842. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2343.096. Standard error: 3.279. Cost per task in USD: 0.462842. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":30.555555555555557,"measurements":[{"observationId":"vals-vibe-code-bench:overall:zai/glm-5.3-flash","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":30.759,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.555555555555557,"participants":73,"methodology":{"de":"Kennung des Quellmodells: zai/glm-5.3-flash. Name des Quellmodells: GLM-5.3-Flash. Anbieter laut Quelle: Zhipu AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 5027.281. Standardfehler: 5.277. Kosten je Aufgabe in USD: 2.348805. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: zai/glm-5.3-flash. Source model name: GLM-5.3-Flash. Provider reported by source: Zhipu AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 5027.281. Standard error: 5.277. Cost per task in USD: 2.348805. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":40,"totalFamilies":71,"sourceCount":4,"coverage":0.46629818594104305,"lowerBound":25.63454725677901,"upperBound":79.0047286626747,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":42.4213898588179,"coveredFamilies":11,"totalFamilies":15,"sourceCount":3,"coverage":0.688888888888889,"lowerBound":29.22362412496345,"upperBound":60.33473523607455,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":46.99061338984318,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":30.208251464899185,"upperBound":65.9225371791849,"provisional":true,"configurationsDiffer":true},"math":{"score":19.026806526806528,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":11.416083916083917,"upperBound":51.416083916083906,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":62.75736458663288,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":41.83824305775526,"upperBound":75.17157639108859,"provisional":true,"configurationsDiffer":true},"agentic":{"score":66.70509326301455,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":33.35254663150727,"upperBound":83.35254663150727,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":13.61111111111111,"upperBound":73.61111111111111,"provisional":true,"configurationsDiffer":true},"finance":{"score":71.54600194631416,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":40.88342968360809,"upperBound":83.74057254075095,"provisional":true,"configurationsDiffer":true},"legal":{"score":70.12119938349446,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":52.59089953762084,"upperBound":77.59089953762084,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":29.901960784313726,"upperBound":96.5686274509804,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":54.54545454545455,"upperBound":54.54545454545455,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":21.31205673758865,"upperBound":83.81205673758865,"provisional":true,"configurationsDiffer":true}}},{"modelId":"google-gemma-3-12b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-3-4b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-4-12b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-4-26b-a4b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-4-31b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-4-e2b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"google-gemma-4-e4b-it","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gpt-4.1","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":35.507246376811594,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:3b608b70-6434-4baa-99ad-45d499703c67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":42.9931972789116,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.507246376811594,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/3b608b70-6434-4baa-99ad-45d499703c67/gpt-4-1/gpt-4-1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 43,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/3b608b70-6434-4baa-99ad-45d499703c67/gpt-4-1/gpt-4-1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: GPT-4.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 43.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":14.473684210526315,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":65.404,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.473684210526315,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 23.14. Standardfehler: 2.402. Kosten je Aufgabe in USD: 0.013852. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 23.14. Standard error: 2.402. Cost per task in USD: 0.013852. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-assistantbench","area":"agentic","label":{"de":"AssistantBench","en":"AssistantBench"},"weight":0.005952380952380952,"coverage":1,"score":25,"measurements":[{"observationId":"curated:hal-assistantbench-browser-use-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/assistantbench","benchmarkId":"hal-assistantbench","benchmarkVersion":"HAL 33-task public set","comparisonCohort":"hal-assistantbench-browser-use-overall","retrievedAt":"2026-08-05","value":17.39,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":25,"participants":5,"methodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Kosten je Aufgabe in USD: 0.4287878787878788. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-317cdcf90f39. Prüfsumme der Quelldaten: 317cdcf90f39889de796dd5ed97633968cfa6fc40917816c0eff382bcad3b325. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Cost per task in USD: 0.4287878787878788. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-317cdcf90f39. Source payload hash: 317cdcf90f39889de796dd5ed97633968cfa6fc40917816c0eff382bcad3b325. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33."},"scoreRunConfiguration":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Harness: Browser-Use. Agent: Browser-Use","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Harness: Browser-Use. Agent: Browser-Use"},"scoreComparisonSetup":{"publicationCohort":"hal-assistantbench-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":36.33,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Kosten je Aufgabe in USD: 0.7887333333333334. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-f727a2342c00. Prüfsumme der Quelldaten: f727a2342c00533df3b6fb542fd48c0dedb2cf072902596e015182af311abb43. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Cost per task in USD: 0.7887333333333334. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-f727a2342c00. Source payload hash: f727a2342c00533df3b6fb542fd48c0dedb2cf072902596e015182af311abb43. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":40,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":24.51,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":40,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Kosten je Aufgabe in USD: 0.07274509803921568. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-12f107898c93. Prüfsumme der Quelldaten: 12f107898c93eeb656847cf8cefab22d2f5c03b4b0236847b3f2af67c76d3bbc. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Cost per task in USD: 0.07274509803921568. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-12f107898c93. Source payload hash: 12f107898c93eeb656847cf8cefab22d2f5c03b4b0236847b3f2af67c76d3bbc. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":40,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":44.95,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":40,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Kosten je Aufgabe in USD: 0.09153094462540717. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-a4664313897b. Prüfsumme der Quelldaten: a4664313897ba347a5426e4ad9b2f08826296ccbbb924580ee91c890e8aad1d8. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Cost per task in USD: 0.09153094462540717. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-a4664313897b. Source payload hash: a4664313897ba347a5426e4ad9b2f08826296ccbbb924580ee91c890e8aad1d8. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":18.33206686930091,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:3b608b70-6434-4baa-99ad-45d499703c67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":61.2138728323699,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.642857142857142,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/3b608b70-6434-4baa-99ad-45d499703c67/gpt-4-1/gpt-4-1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 61,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/3b608b70-6434-4baa-99ad-45d499703c67/gpt-4-1/gpt-4-1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: GPT-4.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 61.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 (Release gpt-4-1, Zeile gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 (release gpt-4-1, row gpt-4-1, UUID 3b608b70-6434-4baa-99ad-45d499703c67, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":72.386,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.02127659574468,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 12. Standardfehler: 1.075. Kosten je Aufgabe in USD: 0.007959. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 12. Standard error: 1.075. Cost per task in USD: 0.007959. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"curated:scale-multinrc-gpt-4.1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":21.23,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 21,23 % ± 2,47. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f4d7bbe027a3. Prüfsumme der Quelldaten: f4d7bbe027a3947492bab11cf3b69d24ee0ddc6f29349f018a8c38ff7477f988. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 21.23% ± 2.47. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f4d7bbe027a3. Source payload hash: f4d7bbe027a3947492bab11cf3b69d24ee0ddc6f29349f018a8c38ff7477f988. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"curated:scale-prbench-finance-gpt-4.1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":34.32,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":6.666666666666667,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 34,32 % ± 0,26. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f861f08548c8. Prüfsumme der Quelldaten: f861f08548c8d2b87a868c37a97bc6f3f1754fcce2f52e33fcd2220a9ae170a1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 34.32% ± 0.26. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f861f08548c8. Source payload hash: f861f08548c8d2b87a868c37a97bc6f3f1754fcce2f52e33fcd2220a9ae170a1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"curated:scale-prbench-legal-gpt-4.1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":36.48,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":6.666666666666667,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 36,48 % ± 0,05. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f80b77a89b26. Prüfsumme der Quelldaten: f80b77a89b269e92e269001d11c79709007df81515cb0fe92607794abdedf255. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 36.48% ± 0.05. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f80b77a89b26. Source payload hash: f80b77a89b269e92e269001d11c79709007df81515cb0fe92607794abdedf255. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"scicode","area":"coding","label":{"de":"SciCode","en":"SciCode"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-scicode-scicode-tool-calling-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scicode","benchmarkId":"hal-scicode","benchmarkVersion":"65-problem main set","comparisonCohort":"hal-scicode-scicode-tool-calling-overall","retrievedAt":"2026-08-05","value":1.54,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":5,"methodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Kosten je Aufgabe in USD: 1.0675384615384615. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-8754d9d767b9. Prüfsumme der Quelldaten: 8754d9d767b9b85166f6161bacf66e86cd2298b0c94fb3b1a160641dfc04932f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Cost per task in USD: 1.0675384615384615. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-8754d9d767b9. Source payload hash: 8754d9d767b9b85166f6161bacf66e86cd2298b0c94fb3b1a160641dfc04932f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65."},"scoreRunConfiguration":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling"},"scoreComparisonSetup":{"publicationCohort":"hal-scicode-scicode-tool-calling-overall","datasetType":"public","harness":"SciCode Tool Calling","agent":"SciCode Tool Calling","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":25,"measurements":[{"observationId":"curated:hal-swebench-verified-mini-swe-agent-overall-gpt-4.1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/swebench_verified_mini","benchmarkId":"hal-swebench-verified-mini","benchmarkVersion":"50-task random verified subset","comparisonCohort":"hal-swebench-verified-mini-swe-agent-overall","retrievedAt":"2026-08-05","value":44,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":25,"participants":5,"methodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SWE-Agent. Agent: SWE-Agent. Kosten je Aufgabe in USD: 7.872999999999999. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-8f0a81ff0abe. Prüfsumme der Quelldaten: 8f0a81ff0abe16d3fde642ed3e5ce4d491bf37046f48a652776067b0f05efab8. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SWE-Agent. Agent: SWE-Agent. Cost per task in USD: 7.872999999999999. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-8f0a81ff0abe. Source payload hash: 8f0a81ff0abe16d3fde642ed3e5ce4d491bf37046f48a652776067b0f05efab8. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50."},"scoreRunConfiguration":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Harness: SWE-Agent. Agent: SWE-Agent","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Harness: SWE-Agent. Agent: SWE-Agent"},"scoreComparisonSetup":{"publicationCohort":"hal-swebench-verified-mini-swe-agent-overall","datasetType":"public","harness":"SWE-Agent","agent":"SWE-Agent","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":4.761904761904762,"measurements":[{"observationId":"vals-terminal-bench-2:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":14.607,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.761904761904762,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 589.207. Standardfehler: 3.765. Kosten je Aufgabe in USD: 1.078347. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 589.207. Standard error: 3.765. Cost per task in USD: 1.078347. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":59.45945945945946,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":63.054,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.45945945945946,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 10.14. Standardfehler: 0.951. Kosten je Aufgabe in USD: 0.13202. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 10.14. Standard error: 0.951. Cost per task in USD: 0.13202. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":11.688311688311689,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":54.666,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.688311688311689,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 30.565. Standardfehler: 1.118. Kosten je Aufgabe in USD: 0.00931. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 30.565. Standard error: 1.118. Cost per task in USD: 0.00931. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":50,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.1,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 0.509. Standardfehler: 0.427. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 0.509. Standard error: 0.427. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":31.818181818181817,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":87.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.818181818181817,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 12.53. Standardfehler: 1.494. Kosten je Aufgabe in USD: 0.009426. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768. Latency in seconds: 12.53. Standard error: 1.494. Cost per task in USD: 0.009426. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":48.717948717948715,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":91.183,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.717948717948715,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 3.083. Standardfehler: 0.259. Kosten je Aufgabe in USD: 0.002952. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 3.083. Standard error: 0.259. Cost per task in USD: 0.002952. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":20.689655172413794,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":87.673,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.689655172413794,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 2.19. Standardfehler: 1.921. Kosten je Aufgabe in USD: 0.001684. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 2.19. Standard error: 1.921. Cost per task in USD: 0.001684. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":25.974025974025974,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":80.495,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.974025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 8.177. Standardfehler: 0.385. Kosten je Aufgabe in USD: 0.007141. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 8.177. Standard error: 0.385. Cost per task in USD: 0.007141. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":48.07692307692308,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.938,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.07692307692308,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 5.034. Standardfehler: 0.922. Kosten je Aufgabe in USD: 0.006212. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 5.034. Standard error: 0.922. Cost per task in USD: 0.006212. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":78.94736842105263,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-4.1-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.061,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.94736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-2025-04-14. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 7.89. Standardfehler: 0.845. Kosten je Aufgabe in USD: 0.006214. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-2025-04-14. Source model name: GPT-4.1. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 7.89. Standard error: 0.845. Cost per task in USD: 0.006214. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"curated:scale-vista-gpt-4.1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":45.34,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 45,34 % ± 0,91. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-c0983d872471. Prüfsumme der Quelldaten: c0983d8724716bd2b4723d28cda3ebee1947978970afd9df043690aed9497d1c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 45.34% ± 0.91. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-c0983d872471. Source payload hash: c0983d8724716bd2b4723d28cda3ebee1947978970afd9df043690aed9497d1c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"curated:scale-visual-toolbench-gpt-4.1","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":5.52,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/gpt-4.1. Name des Quellmodells: GPT-4.1. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 5,52 % ± 0,21. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-3cbede28fae9. Prüfsumme der Quelldaten: 3cbede28fae90810f4803c0ba2a633d2f183b5f3e56e3c439c717735af78062b. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/gpt-4.1. Source model name: GPT-4.1. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 5.52% ± 0.21. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-3cbede28fae9. Source payload hash: 3cbede28fae90810f4803c0ba2a633d2f183b5f3e56e3c439c717735af78062b. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":24,"totalFamilies":71,"sourceCount":4,"coverage":0.2700113378684808,"lowerBound":8.763821850816935,"upperBound":81.76268806396887,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.3111111111111112,"lowerBound":6.998797498797502,"upperBound":75.88768638768637,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":2,"coverage":0.2142857142857143,"lowerBound":2.3325358851674642,"upperBound":80.90396445659603,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":10.501567398119121,"upperBound":70.50156739811912,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":8.658008658008658,"upperBound":75.32467532467533,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":10.416666666666668,"upperBound":93.75000000000001,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":7.101449275362319,"upperBound":87.10144927536231,"provisional":true,"configurationsDiffer":true},"finance":{"score":48.28760440602546,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":27.59291680344312,"upperBound":70.45005966058598,"provisional":true,"configurationsDiffer":true},"legal":{"score":28.333333333333332,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.5,"lowerBound":14.166666666666666,"upperBound":64.16666666666667,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.23931623931624,"upperBound":82.90598290598291,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":24.91410775985244,"coveredFamilies":3,"totalFamilies":4,"sourceCount":3,"coverage":0.75,"lowerBound":18.685580819889328,"upperBound":43.685580819889324,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-4.1-mini","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":23.18840579710145,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:9f7c7566-a704-49a2-a383-cb3181da33a4","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":38.2993197278912,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.18840579710145,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/9f7c7566-a704-49a2-a383-cb3181da33a4/gpt-4-1-mini/gpt-4-1-mini. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1 mini. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 38,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/9f7c7566-a704-49a2-a383-cb3181da33a4/gpt-4-1-mini/gpt-4-1-mini. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: GPT-4.1 mini. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 38.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":15.789473684210526,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":67.929,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.789473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 9.282. Standardfehler: 2.351. Kosten je Aufgabe in USD: 0.002324. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 9.282. Standard error: 2.351. Cost per task in USD: 0.002324. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":14.418693009118542,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:9f7c7566-a704-49a2-a383-cb3181da33a4","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":58.728323699422006,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":16.071428571428573,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/9f7c7566-a704-49a2-a383-cb3181da33a4/gpt-4-1-mini/gpt-4-1-mini. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1 mini. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 58,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/9f7c7566-a704-49a2-a383-cb3181da33a4/gpt-4-1-mini/gpt-4-1-mini. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: GPT-4.1 mini. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 58.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 mini (Release gpt-4-1-mini, Zeile gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 mini (release gpt-4-1-mini, row gpt-4-1-mini, UUID 9f7c7566-a704-49a2-a383-cb3181da33a4, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":70.537,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.76595744680851,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 8.225. Standardfehler: 1.096. Kosten je Aufgabe in USD: 0.001788. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 8.225. Standard error: 1.096. Cost per task in USD: 0.001788. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":22.972972972972972,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":57.926,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.972972972972972,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 6.443. Standardfehler: 0.972. Kosten je Aufgabe in USD: 0.026113. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 6.443. Standard error: 0.972. Cost per task in USD: 0.026113. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":58.158,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 27.592. Standardfehler: 1.181. Kosten je Aufgabe in USD: 0.001178. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 27.592. Standard error: 1.181. Cost per task in USD: 0.001178. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":78.044,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 0.467. Standardfehler: 0.419. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 0.467. Standard error: 0.419. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":88,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 5.76. Standardfehler: 1.453. Kosten je Aufgabe in USD: 0.00122. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768. Latency in seconds: 5.76. Standard error: 1.453. Cost per task in USD: 0.00122. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":84.633,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 1.858. Standardfehler: 0.329. Kosten je Aufgabe in USD: 0.000519. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 1.858. Standard error: 0.329. Cost per task in USD: 0.000519. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":24.137931034482758,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":87.782,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.137931034482758,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 2.748. Standardfehler: 1.928. Kosten je Aufgabe in USD: 0.000394. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 2.748. Standard error: 1.928. Cost per task in USD: 0.000394. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":15.584415584415584,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":77.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.584415584415584,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 3.832. Standardfehler: 0.405. Kosten je Aufgabe in USD: 0.001316. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 3.832. Standard error: 0.405. Cost per task in USD: 0.001316. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":42.30769230769231,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.501,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.30769230769231,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 4.598. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.002797. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 4.598. Standard error: 0.938. Cost per task in USD: 0.002797. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":42.10526315789474,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-4.1-mini-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.914,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.10526315789474,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-mini-2025-04-14. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 5.921. Standardfehler: 0.881. Kosten je Aufgabe in USD: 0.001233. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-mini-2025-04-14. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-mini-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 5.921. Standard error: 0.881. Cost per task in USD: 0.001233. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"curated:scale-vista-gpt-4.1-mini","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":41.14,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":15.384615384615385,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-4.1-mini. Name des Quellmodells: GPT-4.1 mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 41,14 % ± 0,58. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-5edf9af15d2e. Prüfsumme der Quelldaten: 5edf9af15d2e4bb35a694ad24c9d764cfdc08543d3ae92cf5a387efbda259c8c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-4.1-mini. Source model name: GPT-4.1 mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 41.14% ± 0.58. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-5edf9af15d2e. Source payload hash: 5edf9af15d2e4bb35a694ad24c9d764cfdc08543d3ae92cf5a387efbda259c8c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":13,"totalFamilies":71,"sourceCount":3,"coverage":0.18452380952380956,"lowerBound":4.548929730976578,"upperBound":86.09654877859563,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.9523809523809527,"upperBound":94.28571428571428,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.1278195488721805,"upperBound":93.98496240601504,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":12.100313479623825,"upperBound":72.10031347962382,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":5.194805194805195,"upperBound":71.86147186147187,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.63768115942029,"upperBound":84.6376811594203,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":15.340846919794286,"upperBound":72.48370406265143,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":5.769230769230769,"upperBound":80.76923076923077,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":11.111111111111112,"upperBound":77.77777777777779,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":14.901654196866962,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":7.450827098433481,"upperBound":57.45082709843348,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-4.1-nano","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":10.869565217391305,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:72c358fd-7d45-4d68-89aa-699743710924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":32.0408163265306,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.869565217391305,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/72c358fd-7d45-4d68-89aa-699743710924/gpt-4-1-nano/gpt-4-1-nano. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1 nano. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 32,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/72c358fd-7d45-4d68-89aa-699743710924/gpt-4-1-nano/gpt-4-1-nano. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: GPT-4.1 nano. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 32.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":9.210526315789474,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":50.758,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.210526315789474,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 4.192. Standardfehler: 2.457. Kosten je Aufgabe in USD: 0.000488. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 4.192. Standard error: 2.457. Cost per task in USD: 0.000488. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":0.8928571428571429,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:72c358fd-7d45-4d68-89aa-699743710924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":40.1156069364162,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.7857142857142858,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/72c358fd-7d45-4d68-89aa-699743710924/gpt-4-1-nano/gpt-4-1-nano. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4.1 nano. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 40,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/72c358fd-7d45-4d68-89aa-699743710924/gpt-4-1-nano/gpt-4-1-nano. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: GPT-4.1 nano. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 40.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4.1 nano (Release gpt-4-1-nano, Zeile gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4.1 nano (release gpt-4-1-nano, row gpt-4-1-nano, UUID 72c358fd-7d45-4d68-89aa-699743710924, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":55.055,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 7.251. Standardfehler: 1.196. Kosten je Aufgabe in USD: 0.000566. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 7.251. Standard error: 1.196. Cost per task in USD: 0.000566. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":42.075,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 4.954. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.006454. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 4.954. Standard error: 0.968. Cost per task in USD: 0.006454. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":7.792207792207792,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":42.718,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.792207792207792,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 11.801. Standardfehler: 1.152. Kosten je Aufgabe in USD: 0.000298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 11.801. Standard error: 1.152. Cost per task in USD: 0.000298. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":2.5641025641025643,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":61.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.5641025641025643,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 0.357. Standardfehler: 0.595. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 0.357. Standard error: 0.595. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":22.727272727272727,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":80.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.727272727272727,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 3.372. Standardfehler: 1.782. Kosten je Aufgabe in USD: 0.000314. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768. Latency in seconds: 3.372. Standard error: 1.782. Cost per task in USD: 0.000314. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":68.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 1.416. Standardfehler: 0.425. Kosten je Aufgabe in USD: 0.000154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 1.416. Standard error: 0.425. Cost per task in USD: 0.000154. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":69.273,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 1.458. Standardfehler: 2.799. Kosten je Aufgabe in USD: 0.000129. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 1.458. Standard error: 2.799. Cost per task in USD: 0.000129. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":2.5974025974025974,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":63.479,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.5974025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 2.403. Standardfehler: 0.462. Kosten je Aufgabe in USD: 0.000337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 2.403. Standard error: 0.462. Cost per task in USD: 0.000337. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":52.822,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 3.858. Standardfehler: 0.995. Kosten je Aufgabe in USD: 0.001003. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 3.858. Standard error: 0.995. Cost per task in USD: 0.001003. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":3.9473684210526314,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-4.1-nano-2025-04-14","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":60.752,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.9473684210526314,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4.1-nano-2025-04-14. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 2.998. Standardfehler: 0.956. Kosten je Aufgabe in USD: 0.000291. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4.1-nano-2025-04-14. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Model snapshot: gpt-4.1-nano-2025-04-14. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 2.998. Standard error: 0.956. Cost per task in USD: 0.000291. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-vista-gpt-4.1-nano","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":26.55,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-4.1-nano. Name des Quellmodells: GPT-4.1 nano. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 26,55 % ± 0,35. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-a4d5ac758f60. Prüfsumme der Quelldaten: a4d5ac758f60c33e7a59f3f05e6e0ddfe41226fbcd27fb88bced23199c68c38f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-4.1-nano. Source model name: GPT-4.1 nano. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 26.55% ± 0.35. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-a4d5ac758f60. Source payload hash: a4d5ac758f60c33e7a59f3f05e6e0ddfe41226fbcd27fb88bced23199c68c38f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":13,"totalFamilies":71,"sourceCount":3,"coverage":0.18452380952380956,"lowerBound":0.9895497910882841,"upperBound":82.53716883870733,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.5194805194805197,"upperBound":93.85281385281385,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0.6578947368421053,"upperBound":93.51503759398497,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":4.545454545454546,"upperBound":64.54545454545455,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0.8658008658008658,"upperBound":67.53246753246754,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.1739130434782608,"upperBound":82.17391304347827,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":1.6628108733371891,"upperBound":58.805668016194325,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":0.6410256410256411,"upperBound":75.64102564102564,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":2.564102564102564,"upperBound":69.23076923076924,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":0.44642857142857145,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":0.22321428571428573,"upperBound":50.223214285714285,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-4o","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":14.492753623188406,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:c1045dc0-4fd3-4adb-9548-18763e0d051f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":34.285714285714306,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.492753623188406,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4o (Nov '24) (Release gpt-4o, Zeile gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/c1045dc0-4fd3-4adb-9548-18763e0d051f/gpt-4o/gpt-4o. Name des Quellmodells: GPT-4o. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4o (Nov '24). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 34,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4o (Nov '24) (release gpt-4o, row gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/c1045dc0-4fd3-4adb-9548-18763e0d051f/gpt-4o/gpt-4o. Source model name: GPT-4o. Provider reported by source: OpenAI. Model snapshot: GPT-4o (Nov '24). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 34.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4o (Nov '24) (Release gpt-4o, Zeile gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o (Nov '24) (release gpt-4o, row gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4o (Nov '24) (Release gpt-4o, Zeile gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o (Nov '24) (release gpt-4o, row gpt-4o, UUID c1045dc0-4fd3-4adb-9548-18763e0d051f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-tutorbench-gpt-4o","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":36.12,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gpt-4o. Name des Quellmodells: GPT-4o. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 36,12 % ± 0,96. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-2d47f3f45c11. Prüfsumme der Quelldaten: 2d47f3f45c11d5dfb09f242e535afc28a9fd65f5672d1a6a41964d0c2ec8c719. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gpt-4o. Source model name: GPT-4o. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 36.12% ± 0.96. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-2d47f3f45c11. Source payload hash: 2d47f3f45c11d5dfb09f242e535afc28a9fd65f5672d1a6a41964d0c2ec8c719. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-4o-2024-11-20","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":43.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-2024-11-20. Name des Quellmodells: GPT-4o. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-2024-11-20. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 4.347. Standardfehler: 1.123. Kosten je Aufgabe in USD: 0.003999. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-2024-11-20. Source model name: GPT-4o. Provider reported by source: OpenAI. Model snapshot: gpt-4o-2024-11-20. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 4.347. Standard error: 1.123. Cost per task in USD: 0.003999. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":35.8974358974359,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-4o-2024-08-06","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":88.161,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.8974358974359,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-2024-08-06. Name des Quellmodells: GPT-4o. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-2024-08-06. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 3.392. Standardfehler: 0.578. Kosten je Aufgabe in USD: 0.002787. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-2024-08-06. Source model name: GPT-4o. Provider reported by source: OpenAI. Model snapshot: gpt-4o-2024-08-06. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 3.392. Standard error: 0.578. Cost per task in USD: 0.002787. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":4,"totalFamilies":71,"sourceCount":3,"coverage":0.07857142857142858,"lowerBound":1.1050302354650183,"upperBound":93.24788737832216,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.6060606060606063,"upperBound":93.93939393939394,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.8985507246376816,"upperBound":82.89855072463769,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":11.965811965811966,"upperBound":78.63247863247864,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":0,"upperBound":50,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-4o-mini","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":8.695652173913043,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":30.952380952381,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.695652173913043,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6/gpt-4o-mini/gpt-4o-mini. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4o mini. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 31,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6/gpt-4o-mini/gpt-4o-mini. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: GPT-4o mini. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 31.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":3.289473684210526,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":44.192,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.289473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 15.32. Standardfehler: 2.495. Kosten je Aufgabe in USD: 0.000536. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 15.32. Standard error: 2.495. Cost per task in USD: 0.000536. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gpt-4o-mini","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":26.3,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gpt-4o-mini. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-441190f58bf4. Prüfsumme der Quelldaten: 441190f58bf43ee1768f98ba21281cc3c97dbd93e50fdde45d9e816fc9137976. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gpt-4o-mini. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-441190f58bf4. Source payload hash: 441190f58bf43ee1768f98ba21281cc3c97dbd93e50fdde45d9e816fc9137976. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gpt-4o-mini","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":29.5,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gpt-4o-mini. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-d92ebee642b2. Prüfsumme der Quelldaten: d92ebee642b24d08cf3917332b6eb47c5e1332b475553443f7b181dd115c5b91. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gpt-4o-mini. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-d92ebee642b2. Source payload hash: d92ebee642b24d08cf3917332b6eb47c5e1332b475553443f7b181dd115c5b91. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":2.849544072948328,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":41.5028901734104,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.5714285714285716,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6/gpt-4o-mini/gpt-4o-mini. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-4o mini. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 41,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6/gpt-4o-mini/gpt-4o-mini. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: GPT-4o mini. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 41.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-4o mini (Release gpt-4o-mini, Zeile gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: GPT-4o mini (release gpt-4o-mini, row gpt-4o-mini, UUID b5c1c91a-7474-4409-9a9c-9c2ac45d9eb6, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":56.557,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.127659574468085,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 45.016. Standardfehler: 1.191. Kosten je Aufgabe in USD: 0.00037. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 45.016. Standard error: 1.191. Cost per task in USD: 0.00037. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":4.054054054054054,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":45.455,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.054054054054054,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 0.041. Standardfehler: 0.977. Kosten je Aufgabe in USD: 0.008848. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 0.041. Standard error: 0.977. Cost per task in USD: 0.008848. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":1.2987012987012987,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":26.423,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.2987012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 7.364. Standardfehler: 1.129. Kosten je Aufgabe in USD: 0.000298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 7.364. Standard error: 1.129. Cost per task in USD: 0.000298. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":4.545454545454546,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":72.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.545454545454546,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 6.304. Standardfehler: 3.898. Kosten je Aufgabe in USD: 0.00035. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 6.304. Standard error: 3.898. Cost per task in USD: 0.00035. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Reasoning effort: high. Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":12.820512820512821,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":72.436,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.820512820512821,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 2.322. Standardfehler: 0.799. Kosten je Aufgabe in USD: 0.000157. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 2.322. Standard error: 0.799. Cost per task in USD: 0.000157. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":13.793103448275861,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":86.182,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.793103448275861,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 4.033. Standardfehler: 2.133. Kosten je Aufgabe in USD: 0.000187. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 4.033. Standard error: 2.133. Cost per task in USD: 0.000187. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":1.2987012987012987,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":62.735,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.2987012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 5.086. Standardfehler: 0.461. Kosten je Aufgabe in USD: 0.000412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 5.086. Standard error: 0.461. Cost per task in USD: 0.000412. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":11.538461538461538,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":54.492,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.538461538461538,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 6.059. Standardfehler: 1.93. Kosten je Aufgabe in USD: 0.000822. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 6.059. Standard error: 1.93. Cost per task in USD: 0.000822. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":2.6315789473684212,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-4o-mini-2024-07-18","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":60.548,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.6315789473684212,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-4o-mini-2024-07-18. Name des Quellmodells: GPT-4o mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 8.899. Standardfehler: 0.963. Kosten je Aufgabe in USD: 0.000323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-4o-mini-2024-07-18. Source model name: GPT-4o mini. Provider reported by source: OpenAI. Model snapshot: gpt-4o-mini-2024-07-18. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384. Latency in seconds: 8.899. Standard error: 0.963. Cost per task in USD: 0.000323. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":13,"totalFamilies":71,"sourceCount":3,"coverage":0.16071428571428578,"lowerBound":0.9821869519922779,"upperBound":84.91075838056369,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.0865800865800866,"upperBound":93.41991341991341,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0.23496240601503757,"upperBound":93.0921052631579,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":3.6677115987460818,"upperBound":63.66771159874608,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0.4329004329004329,"upperBound":67.09956709956711,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":0,"upperBound":83.33333333333334,"provisional":true,"configurationsDiffer":false},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.7391304347826086,"upperBound":81.73913043478261,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":2.603442077126288,"upperBound":59.74629921998343,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":4.273504273504274,"upperBound":70.94017094017094,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":0.712386018237082,"upperBound":75.71238601823708,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":78.26086956521739,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:48e50f00-1fd1-4acc-b337-61078aa341e6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":73.06122448979589,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.26086956521739,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/48e50f00-1fd1-4acc-b337-61078aa341e6/gpt-5/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 73,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/48e50f00-1fd1-4acc-b337-61078aa341e6/gpt-5/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: GPT-5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 73.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":55.26315789473684,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":85.606,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.26315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 169.716. Standardfehler: 1.764. Kosten je Aufgabe in USD: 0.072708. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 169.716. Standard error: 1.764. Cost per task in USD: 0.072708. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-assistantbench","area":"agentic","label":{"de":"AssistantBench","en":"AssistantBench"},"weight":0.005952380952380952,"coverage":1,"score":75,"measurements":[{"observationId":"curated:hal-assistantbench-browser-use-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/assistantbench","benchmarkId":"hal-assistantbench","benchmarkVersion":"HAL 33-task public set","comparisonCohort":"hal-assistantbench-browser-use-overall","retrievedAt":"2026-08-05","value":35.23,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":75,"participants":5,"methodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 1.2633333333333332. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-b037c99662cf. Prüfsumme der Quelldaten: b037c99662cf1205a80a9c7650f605db0cac75c65b50792d387a36d55e9bb080. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium. Cost per task in USD: 1.2633333333333332. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-b037c99662cf. Source payload hash: b037c99662cf1205a80a9c7650f605db0cac75c65b50792d387a36d55e9bb080. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33."},"scoreRunConfiguration":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-assistantbench-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":60,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":32,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":60,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 2.4543666666666666. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-4d01dc241a10. Prüfsumme der Quelldaten: 4d01dc241a10f780fd54bdb1144f6b99c78fe0f2edfa26564339faa029064ff1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium. Cost per task in USD: 2.4543666666666666. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-4d01dc241a10. Source payload hash: 4d01dc241a10f780fd54bdb1144f6b99c78fe0f2edfa26564339faa029064ff1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":80,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":30.39,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":80,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 0.17901960784313728. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-02996f2b8ff9. Prüfsumme der Quelldaten: 02996f2b8ff940ff8edd458c82f56a7c369a23fa26315f8cce4ffaccf29b850a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: medium. Cost per task in USD: 0.17901960784313728. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-02996f2b8ff9. Source payload hash: 02996f2b8ff940ff8edd458c82f56a7c369a23fa26315f8cce4ffaccf29b850a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: medium","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":69.06,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 0.37990228013029315. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-6c8514acdb83. Prüfsumme der Quelldaten: 6c8514acdb833c98ee9a4fd7e0206a3a7139d4334452da7c4f025adf20559f89. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: medium. Cost per task in USD: 0.37990228013029315. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-6c8514acdb83. Source payload hash: 6c8514acdb833c98ee9a4fd7e0206a3a7139d4334452da7c4f025adf20559f89. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: medium","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":0,"measurements":[{"observationId":"curated:hle-scale-public-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":25.32,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 25,32 % ± 1,70. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-be1d3c509518. Prüfsumme der Quelldaten: be1d3c5095186d9f73167cf4dd72b4602fca1ebfc0813ed3d029e30904f82b99. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 25.32% ± 1.70. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-be1d3c509518. Source payload hash: be1d3c5095186d9f73167cf4dd72b4602fca1ebfc0813ed3d029e30904f82b99. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":40.7484802431611,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:48e50f00-1fd1-4acc-b337-61078aa341e6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":74.2196531791907,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.07142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/48e50f00-1fd1-4acc-b337-61078aa341e6/gpt-5/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 74,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/48e50f00-1fd1-4acc-b337-61078aa341e6/gpt-5/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: GPT-5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 74.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 (High) (Release gpt-5, Zeile gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 (High) (release gpt-5, row gpt-5, UUID 48e50f00-1fd1-4acc-b337-61078aa341e6, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":81.503,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.42553191489362,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 72.114. Standardfehler: 0.934. Kosten je Aufgabe in USD: 0.032532. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 72.114. Standard error: 0.934. Cost per task in USD: 0.032532. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"curated:scale-multichallenge-gpt-5-thinking","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":63.19,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":71.42857142857143,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: thinking. Angezeigter Wert: 63,19 % ± 1,63. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-7b7bcaff80a7. Prüfsumme der Quelldaten: 7b7bcaff80a73b4e3c207bffe7195b1d6dbf03be8f0fba0239ff6219d42bc187. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: thinking. Score display: 63.19% ± 1.63. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-7b7bcaff80a7. Source payload hash: 7b7bcaff80a73b4e3c207bffe7195b1d6dbf03be8f0fba0239ff6219d42bc187. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: thinking","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: thinking"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"thinking","weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"curated:scale-multinrc-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":52.13,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 52,13 % ± 3,01. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-5488e7943207. Prüfsumme der Quelldaten: 5488e794320795f001ba176d093e24e3f96a4ecaedfd67689a3c89702e0b8fe0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 52.13% ± 3.01. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-5488e7943207. Source payload hash: 5488e794320795f001ba176d093e24e3f96a4ecaedfd67689a3c89702e0b8fe0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"curated:scale-prbench-finance-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":51.32,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 51,32 % ± 0,17. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-76dc541fd10c. Prüfsumme der Quelldaten: 76dc541fd10ca8725bebcdb3a069b77a3a3c07b06becd05333c48a75d60dfee4. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 51.32% ± 0.17. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-76dc541fd10c. Source payload hash: 76dc541fd10ca8725bebcdb3a069b77a3a3c07b06becd05333c48a75d60dfee4. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"curated:scale-prbench-legal-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":48.96,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 48,96 % ± 0,03. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-a73f1bd73816. Prüfsumme der Quelldaten: a73f1bd7381627023ae4c602b5d0496833ea4b01daffa8a839d039ef803f59ee. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 48.96% ± 0.03. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-a73f1bd73816. Source payload hash: a73f1bd7381627023ae4c602b5d0496833ea4b01daffa8a839d039ef803f59ee. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"scicode","area":"coding","label":{"de":"SciCode","en":"SciCode"},"weight":0.004761904761904762,"coverage":1,"score":75,"measurements":[{"observationId":"curated:hal-scicode-scicode-tool-calling-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scicode","benchmarkId":"hal-scicode","benchmarkVersion":"65-problem main set","comparisonCohort":"hal-scicode-scicode-tool-calling-overall","retrievedAt":"2026-08-05","value":6.15,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":75,"participants":5,"methodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 2.9772307692307693. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-43b66a7409bd. Prüfsumme der Quelldaten: 43b66a7409bdc2f55d1a67906b7ba4833adfd9f219de18276136cea9a8709df2. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: medium. Cost per task in USD: 2.9772307692307693. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-43b66a7409bd. Source payload hash: 43b66a7409bdc2f55d1a67906b7ba4833adfd9f219de18276136cea9a8709df2. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65."},"scoreRunConfiguration":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: medium","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-scicode-scicode-tool-calling-overall","datasetType":"public","harness":"SciCode Tool Calling","agent":"SciCode Tool Calling","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":1,"score":40.99576271186441,"measurements":[{"observationId":"curated:hal-swebench-verified-mini-swe-agent-overall-gpt-5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/swebench_verified_mini","benchmarkId":"hal-swebench-verified-mini","benchmarkVersion":"50-task random verified subset","comparisonCohort":"hal-swebench-verified-mini-swe-agent-overall","retrievedAt":"2026-08-05","value":46,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":62.5,"participants":5,"methodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 3.2586. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-018a4ac8da36. Prüfsumme der Quelldaten: 018a4ac8da36b104fce59003162f8ef071d976f750d71a5dd2b5603c8b81f348. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: medium. Cost per task in USD: 3.2586. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-018a4ac8da36. Source payload hash: 018a4ac8da36b104fce59003162f8ef071d976f750d71a5dd2b5603c8b81f348. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50."},"scoreRunConfiguration":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: medium","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-swebench-verified-mini-swe-agent-overall","datasetType":"public","harness":"SWE-Agent","agent":"SWE-Agent","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-swebench:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":69,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.491525423728813,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 232.842. Standardfehler: 2.07. Kosten je Aufgabe in USD: 0.27029. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 232.842. Standard error: 2.07. Cost per task in USD: 0.27029. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":35.714285714285715,"measurements":[{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":37.079,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.714285714285715,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 986.102. Standardfehler: 5.149. Kosten je Aufgabe in USD: 0.477064. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 986.102. Standard error: 5.149. Cost per task in USD: 0.477064. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"curated:scale-tutorbench-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":55.33,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":77.77777777777777,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 55,33 % ± 1,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-6ea79e886788. Prüfsumme der Quelldaten: 6ea79e88678843e7fb7692c76524df8ef8c0f7b08419d9a866e21e24c1cc5d5a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 55.33% ± 1.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-6ea79e886788. Source payload hash: 6ea79e88678843e7fb7692c76524df8ef8c0f7b08419d9a866e21e24c1cc5d5a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":47.2972972972973,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.072,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.2972972972973,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 32000. Latenz in Sekunden: 160.705. Standardfehler: 0.961. Kosten je Aufgabe in USD: 0.086116. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 32000. Latency in seconds: 160.705. Standard error: 0.961. Cost per task in USD: 0.086116. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32000","en":"Reasoning effort: high. Maximum output tokens: 32000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32000","en":"Reasoning effort: high. Maximum output tokens: 32000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.911,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.72727272727273,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 150.441. Standardfehler: 0.974. Kosten je Aufgabe in USD: 0.063751. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 150.441. Standard error: 0.974. Cost per task in USD: 0.063751. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":82.05128205128206,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.023,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.05128205128206,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.59. Standardfehler: 0.379. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 18.59. Standard error: 0.379. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":96,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 32.217. Standardfehler: 0.876. Kosten je Aufgabe in USD: 0.020071. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 32.217. Standard error: 0.876. Cost per task in USD: 0.020071. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":77.61194029850746,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.634,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.61194029850746,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 58.268. Standardfehler: 2.098. Kosten je Aufgabe in USD: 0.045858. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 58.268. Standard error: 2.098. Cost per task in USD: 0.045858. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":94.87179487179488,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.317,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.87179487179488,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 35.496. Standardfehler: 0.172. Kosten je Aufgabe in USD: 0.020888. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 35.496. Standard error: 0.172. Cost per task in USD: 0.020888. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":54.411764705882355,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.65,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.411764705882355,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 184.536. Standardfehler: 1.936. Kosten je Aufgabe in USD: 0.1015. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 184.536. Standard error: 1.936. Cost per task in USD: 0.1015. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":89.65517241379311,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":92.836,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.65517241379311,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.353. Standardfehler: 1.559. Kosten je Aufgabe in USD: 0.014278. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 20.353. Standard error: 1.559. Cost per task in USD: 0.014278. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.544,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 37.239. Standardfehler: 0.336. Kosten je Aufgabe in USD: 0.024135. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 37.239. Standard error: 0.336. Cost per task in USD: 0.024135. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":40.38461538461539,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.454,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.38461538461539,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 62.828. Standardfehler: 0.879. Kosten je Aufgabe in USD: 0.028906. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 62.828. Standard error: 0.879. Cost per task in USD: 0.028906. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"vals-poker-agent:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1103.1750450374514,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":9,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 76.03939085804956. Standardfehler: 0. Kosten je Aufgabe in USD: 0.05482942854251021. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 76.03939085804956. Standard error: 0. Cost per task in USD: 0.05482942854251021. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":37.93103448275862,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":43.68,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.93103448275862,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 58.593. Standardfehler: 3.349. Kosten je Aufgabe in USD: 0.032224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 58.593. Standard error: 3.349. Cost per task in USD: 0.032224. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":56.578947368421055,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.385,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.578947368421055,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 78.668. Standardfehler: 0.868. Kosten je Aufgabe in USD: 0.053873. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 78.668. Standard error: 0.868. Cost per task in USD: 0.053873. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":25,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":20.088,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-2025-08-07. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1852.149. Standardfehler: 3.409. Kosten je Aufgabe in USD: 1.525167. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-2025-08-07. Source model name: GPT-5. Provider reported by source: OpenAI. Model snapshot: gpt-5-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 1852.149. Standard error: 3.409. Cost per task in USD: 1.525167. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"curated:scale-vista-gpt-5","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":49.69,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":38.46153846153846,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-5. Name des Quellmodells: GPT-5. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 49,69 % ± 1,03. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-b60b68e8b041. Prüfsumme der Quelldaten: b60b68e8b041572f24066032920d40a8b73d36e26499a4675371f5c5270b8b2c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-5. Source model name: GPT-5. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 49.69% ± 1.03. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-b60b68e8b041. Source payload hash: b60b68e8b041572f24066032920d40a8b73d36e26499a4675371f5c5270b8b2c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":31,"totalFamilies":71,"sourceCount":4,"coverage":0.42511337868480725,"lowerBound":27.995387898577597,"upperBound":85.48405003009688,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":2,"coverage":0.41111111111111126,"lowerBound":26.645027759434548,"upperBound":85.53391664832343,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":2,"coverage":0.2857142857142857,"lowerBound":11.739576213260422,"upperBound":83.16814764183185,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":37.931034482758626,"upperBound":97.93103448275863,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":21.21212121212121,"upperBound":87.87878787878788,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":2,"coverage":0.25,"lowerBound":19.583333333333332,"upperBound":94.58333333333333,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":15.652173913043477,"upperBound":95.65217391304347,"provisional":true,"configurationsDiffer":true},"finance":{"score":57.731881679250115,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":32.98964667385721,"upperBound":75.84678953100007,"provisional":true,"configurationsDiffer":true},"legal":{"score":74.35897435897438,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.5,"lowerBound":37.17948717948719,"upperBound":87.17948717948718,"provisional":true,"configurationsDiffer":true},"health":{"score":75.63183329206157,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":75.63183329206157,"upperBound":75.63183329206157,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":39.605009352349775,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":19.802504676174888,"upperBound":69.80250467617489,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":35.714285714285715,"upperBound":85.71428571428572,"provisional":true,"configurationsDiffer":true},"education":{"score":57.854406130268195,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":57.854406130268195,"upperBound":57.854406130268195,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5-mini","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":428.44,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":72.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 72,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 72.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":6.487466331916088,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 6,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 6.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":39.63414634146341,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":8.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.63414634146341,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 8,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 8.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":766.11,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":85.5072463768116,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":75.4421768707483,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.5072463768116,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 75,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 75.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":-17.316666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":36.8421052631579,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":80.303,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.8421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 67.133. Standardfehler: 1.996. Kosten je Aufgabe in USD: 0.013593. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 67.133. Standard error: 1.996. Cost per task in USD: 0.013593. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":26.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":21.4550509731233,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 21,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 21.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":29.283814589665653,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":70.0578034682081,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.035714285714285,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 70,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 70.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":78.914,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.53191489361702,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 26.703. Standardfehler: 0.98. Kosten je Aufgabe in USD: 0.007152. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 26.703. Standard error: 0.98. Cost per task in USD: 0.007152. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":10.169491525423728,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":60.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.169491525423728,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 187.129. Standardfehler: 2.185. Kosten je Aufgabe in USD: 0.050393. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 187.129. Standard error: 2.185. Cost per task in USD: 0.050393. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":18.619337979094077,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:29855680-7469-43eb-8b88-cd3fb1d99da3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29855680-7469-43eb-8b88-cd3fb1d99da3/gpt-5-mini/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5 mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 mini (High) (Release gpt-5-mini, Zeile gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 mini (High) (release gpt-5-mini, row gpt-5-mini, UUID 29855680-7469-43eb-8b88-cd3fb1d99da3, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":26.966,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.428571428571427,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1028.621. Standardfehler: 4.731. Kosten je Aufgabe in USD: 0.116731. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 1028.621. Standard error: 4.731. Cost per task in USD: 0.116731. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":35.13513513513514,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.179,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.13513513513514,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 73.788. Standardfehler: 0.965. Kosten je Aufgabe in USD: 0.020976. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 73.788. Standard error: 0.965. Cost per task in USD: 0.020976. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":77.92207792207792,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":86.605,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.92207792207792,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 33.673. Standardfehler: 0.979. Kosten je Aufgabe in USD: 0.01153. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 33.673. Standard error: 0.979. Cost per task in USD: 0.01153. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":39.743589743589745,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":81.77,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.743589743589745,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9.432. Standardfehler: 0.452. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 9.432. Standard error: 0.452. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":93.18181818181819,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.18181818181819,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.057. Standardfehler: 0.993. Kosten je Aufgabe in USD: 0.003686. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 12.057. Standard error: 0.993. Cost per task in USD: 0.003686. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":47.76119402985075,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":43.045,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.76119402985075,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 28.173. Standardfehler: 2.045. Kosten je Aufgabe in USD: 0.00556. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 28.173. Standard error: 2.045. Cost per task in USD: 0.00556. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":88.46153846153847,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.058,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.46153846153847,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.253. Standardfehler: 0.178. Kosten je Aufgabe in USD: 0.002953. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 16.253. Standard error: 0.178. Cost per task in USD: 0.002953. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":42.64705882352941,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":80.577,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.64705882352941,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 275.717. Standardfehler: 1.924. Kosten je Aufgabe in USD: 0.033478. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 275.717. Standard error: 1.924. Cost per task in USD: 0.033478. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":86.20689655172414,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":92.582,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.20689655172414,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.573. Standardfehler: 1.59. Kosten je Aufgabe in USD: 0.003698. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 12.573. Standard error: 1.59. Cost per task in USD: 0.003698. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":33.76623376623377,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":82.226,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.76623376623377,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.196. Standardfehler: 0.373. Kosten je Aufgabe in USD: 0.005196. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 22.196. Standard error: 0.373. Cost per task in USD: 0.005196. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":57.69230769230769,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":66.892,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.69230769230769,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 24.864. Standardfehler: 0.923. Kosten je Aufgabe in USD: 0.00436. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 24.864. Standard error: 0.923. Cost per task in USD: 0.00436. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":32.758620689655174,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":42.988,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.758620689655174,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 36.524. Standardfehler: 3.338. Kosten je Aufgabe in USD: 0.005183. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 36.524. Standard error: 3.338. Cost per task in USD: 0.005183. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":81.57894736842105,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.57894736842105,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 37.434. Standardfehler: 0.847. Kosten je Aufgabe in USD: 0.011528. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 37.434. Standard error: 0.847. Cost per task in USD: 0.011528. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5-mini-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":14.171,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.666666666666668,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-mini-2025-08-07. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 698.246. Standardfehler: 3.539. Kosten je Aufgabe in USD: 0.248587. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-mini-2025-08-07. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5-mini-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 698.246. Standard error: 3.539. Cost per task in USD: 0.248587. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"curated:scale-vista-gpt-5-mini","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":50.39,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":53.84615384615385,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-5-mini. Name des Quellmodells: GPT-5 mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 50,39 % ± 2,28. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-b8dc5ab8bbb0. Prüfsumme der Quelldaten: b8dc5ab8bbb009a6fbf690c430bb2212be59d3770af6fa9b122ada8b7d60f0e6. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-5-mini. Source model name: GPT-5 mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 50.39% ± 2.28. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-b8dc5ab8bbb0. Source payload hash: b8dc5ab8bbb009a6fbf690c430bb2212be59d3770af6fa9b122ada8b7d60f0e6. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":26,"totalFamilies":71,"sourceCount":3,"coverage":0.34535147392290255,"lowerBound":17.02395695374624,"upperBound":82.48880956145598,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":4,"totalFamilies":15,"sourceCount":2,"coverage":0.2111111111111112,"lowerBound":7.472425489167947,"upperBound":86.36131437805682,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":14.565376856776089,"upperBound":71.70823399963322,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":35.877742946708466,"upperBound":95.87774294670847,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":35.78555590750713,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":23.85703727167142,"upperBound":57.19037060500476,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":7.825203252032521,"upperBound":82.82520325203252,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":17.10144927536232,"upperBound":97.10144927536231,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":24.915198599409127,"upperBound":82.05805574226626,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":9.935897435897436,"upperBound":84.93589743589743,"provisional":true,"configurationsDiffer":true},"health":{"score":59.62326377163954,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":59.62326377163954,"upperBound":59.62326377163954,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":41.564984217909746,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":20.782492108954873,"upperBound":70.78249210895487,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":16.379310344827587,"upperBound":66.37931034482759,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5-nano","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":59.42028985507246,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:e18e5e6a-5a31-4c0b-b80b-ac401392f446","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":67.5510204081633,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.42028985507246,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/e18e5e6a-5a31-4c0b-b80b-ac401392f446/gpt-5-nano/gpt-5-nano. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 nano (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 67,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/e18e5e6a-5a31-4c0b-b80b-ac401392f446/gpt-5-nano/gpt-5-nano. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5 nano (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 67.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":13.157894736842104,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":63.384,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.157894736842104,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 49.716. Standardfehler: 2.455. Kosten je Aufgabe in USD: 0.004578. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 49.716. Standard error: 2.455. Cost per task in USD: 0.004578. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":16.375379939209726,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:e18e5e6a-5a31-4c0b-b80b-ac401392f446","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":60.982658959537595,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.857142857142858,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/e18e5e6a-5a31-4c0b-b80b-ac401392f446/gpt-5-nano/gpt-5-nano. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5 nano (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 61,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/e18e5e6a-5a31-4c0b-b80b-ac401392f446/gpt-5-nano/gpt-5-nano. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5 nano (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 61.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5 nano (High) (Release gpt-5-nano, Zeile gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: GPT-5 nano (High) (release gpt-5-nano, row gpt-5-nano, UUID e18e5e6a-5a31-4c0b-b80b-ac401392f446, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":70.942,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.893617021276595,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 30.776. Standardfehler: 1.091. Kosten je Aufgabe in USD: 0.002619. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 30.776. Standard error: 1.091. Cost per task in USD: 0.002619. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":22.07792207792208,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":70.216,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.07792207792208,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 136.359. Standardfehler: 0.927. Kosten je Aufgabe in USD: 0.005053. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 136.359. Standard error: 0.927. Cost per task in USD: 0.005053. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":1.2820512820512822,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":50.129,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.2820512820512822,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.496. Standardfehler: 0.472. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 22.496. Standard error: 0.472. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-math500:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":93.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.396. Standardfehler: 1.078. Kosten je Aufgabe in USD: 0.001664. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 22.396. Standard error: 1.078. Cost per task in USD: 0.001664. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":4.477611940298507,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":30.441,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.477611940298507,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 29.741. Standardfehler: 1.948. Kosten je Aufgabe in USD: 0.001729. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 29.741. Standard error: 1.948. Cost per task in USD: 0.001729. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":93.258,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.462. Standardfehler: 0.229. Kosten je Aufgabe in USD: 0.001254. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 20.462. Standard error: 0.229. Cost per task in USD: 0.001254. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":19.11764705882353,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":72.865,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.11764705882353,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 112.909. Standardfehler: 1.891. Kosten je Aufgabe in USD: 0.006961. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 112.909. Standard error: 1.891. Cost per task in USD: 0.006961. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":43.10344827586207,"measurements":[{"observationId":"vals-mgsm:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":89.309,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.10344827586207,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.775. Standardfehler: 1.892. Kosten je Aufgabe in USD: 0.001888. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 22.775. Standard error: 1.892. Cost per task in USD: 0.001888. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":12.987012987012987,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":76.067,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.987012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 26.107. Standardfehler: 0.412. Kosten je Aufgabe in USD: 0.001887. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 26.107. Standard error: 0.412. Cost per task in USD: 0.001887. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":9.615384615384615,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":53.617,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.615384615384615,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 33.364. Standardfehler: 0.969. Kosten je Aufgabe in USD: 0.002295. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 33.364. Standard error: 0.969. Cost per task in USD: 0.002295. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":6.896551724137931,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":30.377,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.896551724137931,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 38.714. Standardfehler: 2.834. Kosten je Aufgabe in USD: 0.002256. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 38.714. Standard error: 2.834. Cost per task in USD: 0.002256. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":13.157894736842104,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5-nano-2025-08-07","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":67.376,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.157894736842104,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5-nano-2025-08-07. Name des Quellmodells: GPT-5 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 69.944. Standardfehler: 0.913. Kosten je Aufgabe in USD: 0.004904. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5-nano-2025-08-07. Source model name: GPT-5 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5-nano-2025-08-07. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000. Latency in seconds: 69.944. Standard error: 0.913. Cost per task in USD: 0.004904. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Ausführlichkeit: medium. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Verbosity: medium. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":14,"totalFamilies":71,"sourceCount":2,"coverage":0.239795918367347,"lowerBound":5.79828597305567,"upperBound":81.81869413632097,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":1.4718614718614724,"upperBound":94.8051948051948,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0.9398496240601503,"upperBound":93.796992481203,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":21.34796238244514,"upperBound":81.34796238244513,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":4.329004329004329,"upperBound":70.995670995671,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.884057971014492,"upperBound":91.8840579710145,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":3.253325621746674,"upperBound":74.6818970503181,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":0.32051282051282054,"upperBound":75.32051282051282,"provisional":true,"configurationsDiffer":true},"health":{"score":30.087308555262904,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":30.087308555262904,"upperBound":30.087308555262904,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":4.093844984802431,"upperBound":79.09384498480243,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":3.4482758620689653,"upperBound":53.44827586206897,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5-pro","contributions":[{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":16.666666666666668,"measurements":[{"observationId":"curated:hle-scale-public-gpt-5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":31.64,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":16.666666666666668,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/gpt-5-pro. Name des Quellmodells: GPT-5 Pro. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5-pro-2025-10-06. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 31,64 % ± 1,82. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-2fa0a1a962e9. Prüfsumme der Quelldaten: 2fa0a1a962e9a3c5220390ba98c86b5502caf5700cda9ca4ebac05222503d874. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/gpt-5-pro. Source model name: GPT-5 Pro. Provider reported by source: OpenAI. Model snapshot: gpt-5-pro-2025-10-06. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 31.64% ± 1.82. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-2fa0a1a962e9. Source payload hash: 2fa0a1a962e9a3c5220390ba98c86b5502caf5700cda9ca4ebac05222503d874. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-multinrc-gpt-5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":65.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gpt-5-pro. Name des Quellmodells: GPT-5 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 65,2 % ± 1,24. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-57a178cbb173. Prüfsumme der Quelldaten: 57a178cbb173288aaa3efcbad6e3216d222f4787159448eaccdff6629d74f8c2. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gpt-5-pro. Source model name: GPT-5 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 65.2% ± 1.24. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-57a178cbb173. Source payload hash: 57a178cbb173288aaa3efcbad6e3216d222f4787159448eaccdff6629d74f8c2. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":80,"measurements":[{"observationId":"curated:scale-prbench-finance-gpt-5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":51.06,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":80,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5-pro. Name des Quellmodells: GPT-5 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 51,06 % ± 0,59. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-466bd0b3fa6e. Prüfsumme der Quelldaten: 466bd0b3fa6e323b6844a677a04942e35399edcb3102716a3c2d0c7338565b09. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5-pro. Source model name: GPT-5 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 51.06% ± 0.59. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-466bd0b3fa6e. Source payload hash: 466bd0b3fa6e323b6844a677a04942e35399edcb3102716a3c2d0c7338565b09. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":80,"measurements":[{"observationId":"curated:scale-prbench-legal-gpt-5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":49.89,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":80,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5-pro. Name des Quellmodells: GPT-5 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 49,89 % ± 0,36. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-08c20ad4ef1f. Prüfsumme der Quelldaten: 08c20ad4ef1fb8e59f1ebfe00288e0e00898bc4db7eca96aa27a92dee3c1664b. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5-pro. Source model name: GPT-5 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 49.89% ± 0.36. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-08c20ad4ef1f. Source payload hash: 08c20ad4ef1fb8e59f1ebfe00288e0e00898bc4db7eca96aa27a92dee3c1664b. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"curated:scale-vista-gpt-5-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":52.39,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-5-pro. Name des Quellmodells: GPT-5 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 52,39 % ± 1,07. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f22f53f59f01. Prüfsumme der Quelldaten: f22f53f59f01af696ad30df581c179307f548cefbd32d346d2fa431d9c25e8dd. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-5-pro. Source model name: GPT-5 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 52.39% ± 1.07. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f22f53f59f01. Source payload hash: f22f53f59f01af696ad30df581c179307f548cefbd32d346d2fa431d9c25e8dd. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":5,"totalFamilies":71,"sourceCount":1,"coverage":0.06122448979591837,"lowerBound":4.861329147043434,"upperBound":98.7388801674516,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-09-02","areas":{"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2142857142857143,"lowerBound":15.476190476190476,"upperBound":94.04761904761905,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":11.428571428571427,"upperBound":97.14285714285715,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":20,"upperBound":95,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":21.153846153846153,"upperBound":96.15384615384616,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.3-codex","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":84.05797101449275,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:59b5b14b-5365-4ee7-824a-18a8e6309644","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":75.3741496598639,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.05797101449275,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/59b5b14b-5365-4ee7-824a-18a8e6309644/gpt-5-3-codex/gpt-5-3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.3 Codex (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 75,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/59b5b14b-5365-4ee7-824a-18a8e6309644/gpt-5-3-codex/gpt-5-3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Model snapshot: GPT-5.3 Codex (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 75.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":60.714285714285715,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:59b5b14b-5365-4ee7-824a-18a8e6309644","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":78.4971098265896,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.714285714285715,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/59b5b14b-5365-4ee7-824a-18a8e6309644/gpt-5-3-codex/gpt-5-3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.3 Codex (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 78,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/59b5b14b-5365-4ee7-824a-18a8e6309644/gpt-5-3-codex/gpt-5-3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Model snapshot: GPT-5.3 Codex (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 78.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.3 Codex (Xhigh) (Release gpt-5-3-codex, Zeile gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.3 Codex (Xhigh) (release gpt-5-3-codex, row gpt-5-3-codex, UUID 59b5b14b-5365-4ee7-824a-18a8e6309644, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":57.6271186440678,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.3-codex","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":78,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.6271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 246.527. Standardfehler: 1.854. Kosten je Aufgabe in USD: 0.455384. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 246.527. Standard error: 1.854. Cost per task in USD: 0.455384. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":88.0952380952381,"measurements":[{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5.3-codex","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":64.045,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.0952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 506.283. Standardfehler: 5.115. Kosten je Aufgabe in USD: 0.345045. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 506.283. Standard error: 5.115. Cost per task in USD: 0.345045. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-5.3-codex","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":53.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1791.685. Standardfehler: 6.312. Kosten je Aufgabe in USD: 2.68083. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1791.685. Standard error: 6.312. Cost per task in USD: 2.68083. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":83.11688311688312,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.3-codex","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.313,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.11688311688312,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 132.631. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.091585. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 132.631. Standard error: 0.968. Cost per task in USD: 0.091585. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":52.77777777777778,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.3-codex","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":61.767,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.77777777777778,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.3-codex. Name des Quellmodells: GPT-5.3-Codex. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4548.115. Standardfehler: 4.714. Kosten je Aufgabe in USD: 11.914536. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.3-codex. Source model name: GPT-5.3-Codex. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 4548.115. Standard error: 4.714. Cost per task in USD: 11.914536. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":7,"totalFamilies":71,"sourceCount":2,"coverage":0.040674603174603176,"lowerBound":2.8136119243406856,"upperBound":98.74615160688037,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":1,"coverage":0.24444444444444452,"lowerBound":14.989687023585335,"upperBound":90.54524257914089,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":16.81159420289855,"upperBound":96.81159420289855,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":7.589285714285714,"upperBound":95.08928571428571,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.4","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":73.07692307692308,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:a89c4b28-2d8c-456e-88ea-255fb51fd2b6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":33.259587020649,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.07692307692308,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 33,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 33.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":82.6086956521739,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:a89c4b28-2d8c-456e-88ea-255fb51fd2b6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":73.9455782312925,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.6086956521739,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 73,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 73.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":78.28947368421052,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":91.666,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.28947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 202.953. Standardfehler: 1.908. Kosten je Aufgabe in USD: 0.20934. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 202.953. Standard error: 1.908. Cost per task in USD: 0.20934. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":66.66666666666667,"measurements":[{"observationId":"curated:hle-scale-public-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":36.24,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 36,24 % ± 1,88. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-a9fe189c5035. Prüfsumme der Quelldaten: a9fe189c5035ea97b0d66bebcfb9b49b59752be18f9d385d18cfb574f9fc4470. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 36.24% ± 1.88. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-a9fe189c5035. Source payload hash: a9fe189c5035ea97b0d66bebcfb9b49b59752be18f9d385d18cfb574f9fc4470. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-5.4","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1401.7662087027077,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":11.11111111111111,"participants":37,"methodology":{"de":"Arena-Bewertung aus 3.990 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 3,990 published pairwise votes.. Source model ID: lmarena/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 3.990 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 3,990 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 3.990 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 3,990 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":65.63449848024317,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:a89c4b28-2d8c-456e-88ea-255fb51fd2b6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":78.4393063583815,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.92857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 78,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/a89c4b28-2d8c-456e-88ea-255fb51fd2b6/gpt-5-4/gpt-5-4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 78.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 (Xhigh) (Release gpt-5-4, Zeile gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 (Xhigh) (release gpt-5-4, row gpt-5-4, UUID a89c4b28-2d8c-456e-88ea-255fb51fd2b6, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":87.514,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.34042553191489,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 65.565. Standardfehler: 0.795. Kosten je Aufgabe in USD: 0.047083. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 65.565. Standard error: 0.795. Cost per task in USD: 0.047083. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"curated:scale-multinrc-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":58.29,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":72.72727272727273,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-xhigh. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: xhigh. Angezeigter Wert: 58,29 % ± 2,98. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-e70f6234a9dc. Prüfsumme der Quelldaten: e70f6234a9dce6f5e31168cb9f350e5c1f69f0fa80a2af48e138742f1c46cc3f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-xhigh. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: xhigh. Score display: 58.29% ± 2.98. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-e70f6234a9dc. Source payload hash: e70f6234a9dce6f5e31168cb9f350e5c1f69f0fa80a2af48e138742f1c46cc3f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1. Reasoning-Stufe: xhigh","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"curated:scale-prbench-finance-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":45.63,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":46.666666666666664,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-high. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 45,63 % ± 0,56. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-58e97b3901b0. Prüfsumme der Quelldaten: 58e97b3901b0c69c30c948a9b446f95f9547a5b8b660177e3421401be6cf9a38. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-high. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 45.63% ± 0.56. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-58e97b3901b0. Source payload hash: 58e97b3901b0c69c30c948a9b446f95f9547a5b8b660177e3421401be6cf9a38. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: high","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"curated:scale-prbench-legal-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":44.35,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":53.333333333333336,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-high. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 44,35 % ± 0,28. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-7c5087fadf0d. Prüfsumme der Quelldaten: 7c5087fadf0d5ab154cecaa3e2d01edefe33305f4dd3ca3aa3520beb3a3b8359. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-high. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 44.35% ± 0.28. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-7c5087fadf0d. Source payload hash: 7c5087fadf0d5ab154cecaa3e2d01edefe33305f4dd3ca3aa3520beb3a3b8359. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: high","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":60.16949152542373,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":78.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.16949152542373,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 307.118. Standardfehler: 1.848. Kosten je Aufgabe in USD: 0.801134. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 307.118. Standard error: 1.848. Cost per task in USD: 0.801134. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":79.76190476190476,"measurements":[{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":58.427,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.76190476190476,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1027.517. Standardfehler: 5.254. Kosten je Aufgabe in USD: 0.511959. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 1027.517. Standard error: 5.254. Cost per task in USD: 0.511959. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":56.45161290322581,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":34.984,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.45161290322581,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2146.253. Standardfehler: 4.113. Kosten je Aufgabe in USD: 7.756561. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2146.253. Standard error: 4.113. Cost per task in USD: 7.756561. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":71.62162162162163,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":65.268,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.62162162162163,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 82.34. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.235266. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 82.34. Standard error: 0.938. Cost per task in USD: 0.235266. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":63.63636363636363,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":84.141,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.63636363636363,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 209.424. Standardfehler: 1.04. Kosten je Aufgabe in USD: 0.329246. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 209.424. Standard error: 1.04. Cost per task in USD: 0.329246. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 876.217. Standardfehler: 0. Kosten je Aufgabe in USD: 3.598304. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 876.217. Standard error: 0. Cost per task in USD: 3.598304. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.044,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 27.787. Standardfehler: 0.414. Kosten je Aufgabe in USD: 0.023053. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 27.787. Standard error: 0.414. Cost per task in USD: 0.023053. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":41.791044776119406,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":41.292,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.791044776119406,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 187.265. Standardfehler: 2.148. Kosten je Aufgabe in USD: 0.212108. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 187.265. Standard error: 2.148. Cost per task in USD: 0.212108. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"vals-medqa:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.092,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.3076923076923,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.784. Standardfehler: 0.177. Kosten je Aufgabe in USD: 0.043618. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 22.784. Standard error: 0.177. Cost per task in USD: 0.043618. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":33.8235294117647,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":77.549,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.8235294117647,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 282.914. Standardfehler: 3.316. Kosten je Aufgabe in USD: 0.639282. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 282.914. Standard error: 3.316. Cost per task in USD: 0.639282. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":74.02597402597402,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.482,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.02597402597402,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 28.875. Standardfehler: 0.421. Kosten je Aufgabe in USD: 0.050888. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 28.875. Standard error: 0.421. Cost per task in USD: 0.050888. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.323,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 45.002. Standardfehler: 0.915. Kosten je Aufgabe in USD: 0.103445. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 45.002. Standard error: 0.915. Cost per task in USD: 0.103445. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2386.411. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2386.411. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":36.206896551724135,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":43.312,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.206896551724135,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 152.404. Standardfehler: 3.115. Kosten je Aufgabe in USD: 0.098204. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 152.404. Standard error: 3.115. Cost per task in USD: 0.098204. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":51.724137931034484,"measurements":[{"observationId":"vals-skillsbench:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":51.711,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.724137931034484,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 542.281. Standardfehler: 4.721. Kosten je Aufgabe in USD: 2.044198. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 542.281. Standard error: 4.721. Cost per task in USD: 2.044198. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":60.526315789473685,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.958,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 78.717. Standardfehler: 0.868. Kosten je Aufgabe in USD: 0.157344. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 78.717. Standard error: 0.868. Cost per task in USD: 0.157344. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":59.72222222222222,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.4-2026-03-05","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":67.421,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.72222222222222,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-2026-03-05. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-2026-03-05. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5174.848. Standardfehler: 4.843. Kosten je Aufgabe in USD: 10.686976. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-2026-03-05. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-2026-03-05. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 5174.848. Standard error: 4.843. Cost per task in USD: 10.686976. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"curated:scale-vista-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":50.89,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-xhigh. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: xhigh. Angezeigter Wert: 50,89 % ± 1,76. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-6adc63390c67. Prüfsumme der Quelldaten: 6adc63390c67ec86f1cb4b96c40a430d351ac4dd567a62370b3480324d4b8596. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-xhigh. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: xhigh. Score display: 50.89% ± 1.76. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-6adc63390c67. Source payload hash: 6adc63390c67ec86f1cb4b96c40a430d351ac4dd567a62370b3480324d4b8596. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1. Reasoning-Stufe: xhigh","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-visual-toolbench-gpt-5.4","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":29.17,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/gpt-5.4. Name des Quellmodells: GPT-5.4. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-high. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 29,17 % ± 0,13. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-e913cc4e147b. Prüfsumme der Quelldaten: e913cc4e147b0919229c371e278ce241eb4813a13914a78e9cdad50b5302cae6. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/gpt-5.4. Source model name: GPT-5.4. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-high. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 29.17% ± 0.13. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-e913cc4e147b. Source payload hash: e913cc4e147b0919229c371e278ce241eb4813a13914a78e9cdad50b5302cae6. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1. Reasoning-Stufe: high","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":28,"totalFamilies":71,"sourceCount":4,"coverage":0.35249433106575956,"lowerBound":20.96814382049036,"upperBound":85.71871071391442,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":2,"coverage":0.3777777777777779,"lowerBound":17.25263348984352,"upperBound":79.47485571206573,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":2,"coverage":0.2857142857142857,"lowerBound":20.74362041467305,"upperBound":92.17219184324448,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":24.675324675324674,"upperBound":91.34199134199135,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":2,"coverage":0.16666666666666666,"lowerBound":10.400088417329798,"upperBound":93.73342175066314,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":16.52173913043478,"upperBound":96.52173913043478,"provisional":true,"configurationsDiffer":true},"finance":{"score":63.45365101944049,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":36.259229153966,"upperBound":79.11637201110887,"provisional":true,"configurationsDiffer":true},"legal":{"score":49.107468123861565,"coveredFamilies":3,"totalFamilies":4,"sourceCount":2,"coverage":0.75,"lowerBound":36.830601092896174,"upperBound":61.830601092896174,"provisional":true,"configurationsDiffer":true},"health":{"score":55.9740888318588,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":55.9740888318588,"upperBound":55.9740888318588,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":75.7243200062349,"coveredFamilies":3,"totalFamilies":4,"sourceCount":3,"coverage":0.75,"lowerBound":56.79324000467618,"upperBound":81.79324000467618,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":18.103448275862068,"upperBound":68.10344827586206,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.4-mini","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":15.789473684210526,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":15,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.789473684210526,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 15,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 15.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":719.36,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":77,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 77,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 77.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":57.69230769230769,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":28.1710914454277,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.69230769230769,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 28,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 28.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":57.31707317073171,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":26.204887225711754,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.31707317073171,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 26,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 26.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":13.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 13,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 13.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1011.97,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":79.71014492753623,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":73.265306122449,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.71014492753623,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 73,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 73.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":31.428571428571427,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":35.2165725047081,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.428571428571427,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 35,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 35.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":-18.916666666666668,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":43.421052631578945,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":83.08,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.421052631578945,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 63.11. Standardfehler: 2.457. Kosten je Aufgabe in USD: 0.012743. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 63.11. Standard error: 2.457. Cost per task in USD: 0.012743. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":28.127896200185397,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 28,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 28.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":8.333333333333334,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-5.4-mini-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1397.5681840162945,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":8.333333333333334,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.978 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.4-mini-high. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,978 published pairwise votes.. Source model ID: lmarena/gpt-5.4-mini-high. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.978 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 11,978 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.978 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 11,978 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"lmarena:overall:gpt-5.4-mini-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1447.1486873786143,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":16.666666666666668,"participants":37,"methodology":{"de":"Arena-Bewertung aus 63.461 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.4-mini-high. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 63,461 published pairwise votes.. Source model ID: lmarena/gpt-5.4-mini-high. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 63.461 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 63,461 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 63.461 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 63,461 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":20,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-5.4-mini-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1250.9376338780971,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":20,"participants":31,"methodology":{"de":"Arena-Bewertung aus 25.733 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.4-mini-high. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 25,733 published pairwise votes.. Source model ID: lmarena/gpt-5.4-mini-high. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 25.733 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 25,733 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 25.733 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 25,733 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":31.68693009118541,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":73.2947976878613,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.714285714285715,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 73,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 73.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":79.249,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.659574468085108,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 50.813. Standardfehler: 0.975. Kosten je Aufgabe in USD: 0.007061. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 50.813. Standard error: 0.975. Cost per task in USD: 0.007061. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":35.59322033898305,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":73,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.59322033898305,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 326.46. Standardfehler: 1.987. Kosten je Aufgabe in USD: 0.511052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 326.46. Standard error: 1.987. Cost per task in USD: 0.511052. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":45.60562304464744,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:ba242e40-83b7-4cd3-a0e0-b56237984914","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":2.02020202020202,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 2,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ba242e40-83b7-4cd3-a0e0-b56237984914/gpt-5-4-mini/gpt-5-4-mini. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 mini (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 2.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 mini (Xhigh) (Release gpt-5-4-mini, Zeile gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 mini (Xhigh) (release gpt-5-4-mini, row gpt-5-4-mini, UUID ba242e40-83b7-4cd3-a0e0-b56237984914, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":44.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.333333333333336,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2241.556. Standardfehler: 4.492. Kosten je Aufgabe in USD: 0.914107. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 2241.556. Standard error: 4.492. Cost per task in USD: 0.914107. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":54.682,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.142857142857146,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 673.036. Standardfehler: 4.556. Kosten je Aufgabe in USD: 0.286739. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 673.036. Standard error: 4.556. Cost per task in USD: 0.286739. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":2.525,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.864864864864865,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1770.424. Standardfehler: 1.01. Kosten je Aufgabe in USD: 1.438224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1770.424. Standard error: 1.01. Cost per task in USD: 1.438224. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":22.580645161290324,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":12.936,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.580645161290324,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1733.471. Standardfehler: 3.641. Kosten je Aufgabe in USD: 1.953198. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1733.471. Standard error: 3.641. Cost per task in USD: 1.953198. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":43.24324324324324,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.917,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.24324324324324,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 52.297. Standardfehler: 0.962. Kosten je Aufgabe in USD: 0.076368. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 52.297. Standard error: 0.962. Cost per task in USD: 0.076368. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":25.423728813559322,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":45.426,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.423728813559322,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2601.215. Standardfehler: 3.645. Kosten je Aufgabe in USD: 2.146984. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2601.215. Standard error: 3.645. Cost per task in USD: 2.146984. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":45.36,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2153.503. Standardfehler: 0.447. Kosten je Aufgabe in USD: 1.201815. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2153.503. Standard error: 0.447. Cost per task in USD: 1.201815. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":41.55844155844156,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.465,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.55844155844156,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 188.757. Standardfehler: 1.094. Kosten je Aufgabe in USD: 0.032935. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 188.757. Standard error: 1.094. Cost per task in USD: 0.032935. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 681.27. Standardfehler: 0. Kosten je Aufgabe in USD: 1.001493. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 681.27. Standard error: 0. Cost per task in USD: 1.001493. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":12.5,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.333333333333334,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2256.339. Standardfehler: 2.299. Kosten je Aufgabe in USD: 1.864673. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2256.339. Standard error: 2.299. Cost per task in USD: 1.864673. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":44.15584415584416,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":84.554,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.15584415584416,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 20.001. Standardfehler: 0.359. Kosten je Aufgabe in USD: 0.00368. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 20.001. Standard error: 0.359. Cost per task in USD: 0.00368. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":26.923076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":63.514,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.923076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 126.454. Standardfehler: 0.906. Kosten je Aufgabe in USD: 0.044769. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 126.454. Standard error: 0.906. Cost per task in USD: 0.044769. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3225.344. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 3225.344. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":81.03448275862068,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":50.813,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.03448275862068,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 141.066. Standardfehler: 3.403. Kosten je Aufgabe in USD: 0.075919. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 141.066. Standard error: 3.403. Cost per task in USD: 0.075919. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":8.808,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1006.185. Standardfehler: 2.045. Kosten je Aufgabe in USD: 0.806171. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1006.185. Standard error: 2.045. Cost per task in USD: 0.806171. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":32.89473684210526,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.218,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.89473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 44.578. Standardfehler: 0.9. Kosten je Aufgabe in USD: 0.00677. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 44.578. Standard error: 0.9. Cost per task in USD: 0.00677. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":45.833333333333336,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.4-mini-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":47.969,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.833333333333336,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-mini-2026-03-17. Name des Quellmodells: GPT-5.4 mini. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2055.117. Standardfehler: 5.606. Kosten je Aufgabe in USD: 1.191299. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-mini-2026-03-17. Source model name: GPT-5.4 mini. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-mini-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2055.117. Standard error: 5.606. Cost per task in USD: 1.191299. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":32,"totalFamilies":71,"sourceCount":3,"coverage":0.3418367346938776,"lowerBound":13.271891144815553,"upperBound":79.0882176754278,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.43333333333333346,"lowerBound":13.746518834593662,"upperBound":70.41318550126032,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":21.307078672290483,"upperBound":78.44993581514763,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":38.54133671206842,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":25.69422447471228,"upperBound":59.027557808045614,"provisional":true,"configurationsDiffer":true},"agentic":{"score":41.26554660572632,"coveredFamilies":6,"totalFamilies":12,"sourceCount":1,"coverage":0.5,"lowerBound":20.63277330286316,"upperBound":70.63277330286316,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":19.275362318840582,"upperBound":79.27536231884058,"provisional":true,"configurationsDiffer":true},"finance":{"score":29.99998955618938,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":25.71427676244804,"upperBound":39.99999104816233,"provisional":true,"configurationsDiffer":true},"legal":{"score":11.994535519125684,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":5.997267759562842,"upperBound":55.997267759562845,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":25.8434650455927,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":12.92173252279635,"upperBound":62.92173252279635,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":40.51724137931034,"upperBound":90.51724137931035,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.4-nano","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":35.36585365853659,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":673.76,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.36585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":44.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":76.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":44.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 76,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 76.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":24.9262536873156,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.15384615384615,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 24,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 24.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":5.9606130748688075,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 6,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 6.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":37.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":7.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 7,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 7.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":947.98,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":89.85507246376811,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":75.9183673469388,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.85507246376811,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 75,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 75.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":8.571428571428571,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":24.3879472693032,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.571428571428571,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 24,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 24.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":-29.466666666666665,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":28.94736842105263,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":77.526,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.94736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 29.498. Standardfehler: 2.417. Kosten je Aufgabe in USD: 0.001565. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 29.498. Standard error: 2.417. Cost per task in USD: 0.001565. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":28.2669138090825,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 28,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 28.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":22.967325227963528,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":65.3757225433526,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.785714285714285,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 65,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 65.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":73.584,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.148936170212767,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 21.907. Standardfehler: 1.06. Kosten je Aufgabe in USD: 0.000844. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 21.907. Standard error: 1.06. Cost per task in USD: 0.000844. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":23.728813559322035,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":69.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.728813559322035,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 262.552. Standardfehler: 2.055. Kosten je Aufgabe in USD: 0.097712. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 262.552. Standard error: 2.055. Cost per task in USD: 0.097712. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":34.9883855981417,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31/gpt-5-4-nano/gpt-5-4-nano. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: GPT-5.4 nano (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.4 nano (Xhigh) (Release gpt-5-4-nano, Zeile gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.4 nano (Xhigh) (release gpt-5-4-nano, row gpt-5-4-nano, UUID d4fc3f33-f2b0-4da1-88ee-f1f82bd4de31, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":39.888,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.23809523809524,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2527.932. Standardfehler: 4.454. Kosten je Aufgabe in USD: 0.224388. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 2527.932. Standard error: 4.454. Cost per task in USD: 0.224388. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":41.573,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.142857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 757.485. Standardfehler: 2.973. Kosten je Aufgabe in USD: 0.072321. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 757.485. Standard error: 2.973. Cost per task in USD: 0.072321. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":30.64516129032258,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":14.467,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.64516129032258,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1594.162. Standardfehler: 4.025. Kosten je Aufgabe in USD: 0.416225. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 1594.162. Standard error: 4.025. Cost per task in USD: 0.416225. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":49.32432432432432,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.189,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.32432432432432,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9.397. Standardfehler: 0.961. Kosten je Aufgabe in USD: 0.01612. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 9.397. Standard error: 0.961. Cost per task in USD: 0.01612. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":23.728813559322035,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":44.751,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.728813559322035,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1767.043. Standardfehler: 3.121. Kosten je Aufgabe in USD: 0.57312. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 1767.043. Standard error: 3.121. Cost per task in USD: 0.57312. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":18.333333333333332,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":38.217,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 334.538. Standardfehler: 1.185. Kosten je Aufgabe in USD: 0.15925. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 334.538. Standard error: 1.185. Cost per task in USD: 0.15925. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":62.33766233766234,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":84.009,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.33766233766234,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 60.089. Standardfehler: 1.044. Kosten je Aufgabe in USD: 0.002468. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 60.089. Standard error: 1.044. Cost per task in USD: 0.002468. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 363.008. Standardfehler: 0. Kosten je Aufgabe in USD: 0.184287. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 363.008. Standard error: 0. Cost per task in USD: 0.184287. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":21.794871794871796,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":77.92,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.794871794871796,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2.697. Standardfehler: 0.427. Kosten je Aufgabe in USD: 0.000114. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 2.697. Standard error: 0.427. Cost per task in USD: 0.000114. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":6.25,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.666666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 526.838. Standardfehler: 1.682. Kosten je Aufgabe in USD: 0.141253. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 526.838. Standard error: 1.682. Cost per task in USD: 0.141253. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":37.3134328358209,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":41.029,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.3134328358209,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 8.043. Standardfehler: 2.256. Kosten je Aufgabe in USD: 0.000844. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 8.043. Standard error: 2.256. Cost per task in USD: 0.000844. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":32.35294117647059,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":77.09,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.35294117647059,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 20.527. Standardfehler: 1.891. Kosten je Aufgabe in USD: 0.0018. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 20.527. Standard error: 1.891. Cost per task in USD: 0.0018. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":77.172,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8.21. Standardfehler: 0.433. Kosten je Aufgabe in USD: 0.000396. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 8.21. Standard error: 0.433. Cost per task in USD: 0.000396. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":59.102,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.384615384615385,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 14.526. Standardfehler: 0.964. Kosten je Aufgabe in USD: 0.002585. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 14.526. Standard error: 0.964. Cost per task in USD: 0.002585. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":20.689655172413794,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":38.081,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.689655172413794,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 34.334. Standardfehler: 3.087. Kosten je Aufgabe in USD: 0.005936. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 34.334. Standard error: 3.087. Cost per task in USD: 0.005936. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":7.2727272727272725,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":4.974,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.2727272727272725,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 220.306. Standardfehler: 1.417. Kosten je Aufgabe in USD: 0.069662. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 220.306. Standard error: 1.417. Cost per task in USD: 0.069662. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":14.473684210526315,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":67.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.473684210526315,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15.898. Standardfehler: 0.926. Kosten je Aufgabe in USD: 0.000638. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 15.898. Standard error: 0.926. Cost per task in USD: 0.000638. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":29.166666666666668,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.4-nano-2026-03-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":26.097,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.166666666666668,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.4-nano-2026-03-17. Name des Quellmodells: GPT-5.4 nano. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3276.224. Standardfehler: 5.075. Kosten je Aufgabe in USD: 1.277298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.4-nano-2026-03-17. Source model name: GPT-5.4 nano. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-nano-2026-03-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 3276.224. Standard error: 5.075. Cost per task in USD: 1.277298. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":30,"totalFamilies":71,"sourceCount":2,"coverage":0.3589002267573697,"lowerBound":10.060424963390396,"upperBound":74.17040228765342,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.288888888888889,"lowerBound":10.878059004850938,"upperBound":81.98917011596204,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":16.527599486521183,"upperBound":73.67045662937832,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":21.167247386759584,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":14.111498257839722,"upperBound":47.444831591173056,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":13.910033056374518,"upperBound":72.24336638970784,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":17.97101449275362,"upperBound":97.97101449275362,"provisional":true,"configurationsDiffer":true},"finance":{"score":21.41958301414144,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":18.359642583549807,"upperBound":32.6453568692641,"provisional":true,"configurationsDiffer":true},"legal":{"score":13.039092055485499,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":9.779319041614125,"upperBound":34.77931904161412,"provisional":true,"configurationsDiffer":true},"health":{"score":34.83318700614574,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":23.22212467076383,"upperBound":56.55545800409716,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":5.741831306990882,"upperBound":80.74183130699089,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":10.344827586206897,"upperBound":60.3448275862069,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.4-pro","contributions":[{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":83.33333333333333,"measurements":[{"observationId":"curated:hle-scale-public-gpt-5.4-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/humanitys_last_exam","benchmarkId":"humanitys-last-exam","benchmarkVersion":"final 2,500-question public set","comparisonCohort":"hle-final-scale-public-temperature-0","retrievedAt":"2026-09-02","value":44.32,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":7,"methodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Kennung des Quellmodells: curated/gpt-5.4-pro. Name des Quellmodells: GPT-5.4 Pro. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.4-pro-2026-03-05. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 44,32 % ± 1,95. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-09-02-51f8ddc5f531. Prüfsumme der Quelldaten: 51f8ddc5f531730f2a8577b861c4774e607ed760c25f52108bade0a86a2b5799. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Source model ID: curated/gpt-5.4-pro. Source model name: GPT-5.4 Pro. Provider reported by source: OpenAI. Model snapshot: gpt-5.4-pro-2026-03-05. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 44.32% ± 1.95. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-09-02-51f8ddc5f531. Source payload hash: 51f8ddc5f531730f2a8577b861c4774e607ed760c25f52108bade0a86a2b5799. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor."},"scoreRunConfiguration":{"de":"Alle 2.500 öffentlichen Fragen, Temperatur 0 und automatischer Antwort-Extraktor.. Versuche: pass@1","en":"All 2,500 public questions, temperature 0, and an automatic answer extractor.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"hle-final-scale-public-temperature-0","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":false}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"curated:scale-multichallenge-gpt-5.4-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":69.23,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/gpt-5.4-pro. Name des Quellmodells: GPT-5.4 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 69,23 % ± 3,05. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-094a8bd3e75b. Prüfsumme der Quelldaten: 094a8bd3e75b53ac5a174f8cb8b658af01554eea62ef241f0bf2527433f3849a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/gpt-5.4-pro. Source model name: GPT-5.4 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 69.23% ± 3.05. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-094a8bd3e75b. Source payload hash: 094a8bd3e75b53ac5a174f8cb8b658af01554eea62ef241f0bf2527433f3849a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"curated:scale-multinrc-gpt-5.4-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":62.27,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/gpt-5.4-pro. Name des Quellmodells: GPT-5.4 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 62,27 % ± 2,92. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f6d5c3c0afb5. Prüfsumme der Quelldaten: f6d5c3c0afb57ae872fdc0f7d1a0f6ce1ac051dce7e697faf6e210c8757daa5f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/gpt-5.4-pro. Source model name: GPT-5.4 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 62.27% ± 2.92. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f6d5c3c0afb5. Source payload hash: f6d5c3c0afb57ae872fdc0f7d1a0f6ce1ac051dce7e697faf6e210c8757daa5f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":100,"measurements":[{"observationId":"curated:scale-tutorbench-gpt-5.4-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":56.62,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":100,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/gpt-5.4-pro. Name des Quellmodells: GPT-5.4 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 56,62 % ± 1,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-0a9a42605858. Prüfsumme der Quelldaten: 0a9a4260585830c804e54bb75c5da6d84a438a201917cb239718d094f674c97b. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/gpt-5.4-pro. Source model name: GPT-5.4 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 56.62% ± 1.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-0a9a42605858. Source payload hash: 0a9a4260585830c804e54bb75c5da6d84a438a201917cb239718d094f674c97b. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"curated:scale-vista-gpt-5.4-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":53.89,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":92.3076923076923,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/gpt-5.4-pro. Name des Quellmodells: GPT-5.4 Pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 53,89 % ± 2,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-665228705bb0. Prüfsumme der Quelldaten: 665228705bb014f3400e35f1f04afc38ea636267a546fb05ccd1861bceda080a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/gpt-5.4-pro. Source model name: GPT-5.4 Pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 53.89% ± 2.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-665228705bb0. Source payload hash: 665228705bb014f3400e35f1f04afc38ea636267a546fb05ccd1861bceda080a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":5,"totalFamilies":71,"sourceCount":1,"coverage":0.10459183673469387,"lowerBound":9.541054183911328,"upperBound":99.08187051044195,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-09-02","areas":{"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2142857142857143,"lowerBound":17.640692640692638,"upperBound":96.2121212121212,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":23.076923076923077,"upperBound":98.07692307692308,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":42.857142857142854,"upperBound":92.85714285714286,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":50,"upperBound":100,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":4.545454545454546,"measurements":[{"observationId":"research:gpt-5.5:39","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.8444444444444444,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":4.545454545454546,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_low. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (low). Im Alias kodierte Reasoning-Stufe: low. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: CdmtwZ8NR8YeTXWiLVTRDJ. Startzeit: 2026-08-07T00:39:05.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.8444444444444444 und stderr=0.05463890236888291. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: low.","en":"Independent evaluation. Epoch run alias: gpt-5.5_low; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (low); Reasoning setting encoded in the alias: low. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: CdmtwZ8NR8YeTXWiLVTRDJ. Started at 2026-08-07T00:39:05.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.8444444444444444 and stderr=0.05463890236888291. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: low."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: low. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: low. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"low","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":77.63157894736842,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":50,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":77.63157894736842,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 50,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 50.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1136.33,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":84.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 84,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 84.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":37.6843657817109,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.61538461538461,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 37,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 37.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":47.300565872933134,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 47,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 47.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":21.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 21,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 21.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1350.71,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":88.40579710144928,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":75.8503401360544,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.40579710144928,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 75,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 75.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":68.57142857142857,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":45.8097928436911,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.57142857142857,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 45,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 45.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":20.516666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"research:gpt-5.5:37","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.63,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":85.71428571428571,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_xhigh. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: DFWQcrbMegFAcEWZKWoJxC. Startzeit: 2026-10-13T13:54:48.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.63 und stderr=0.048523658709390974. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: gpt-5.5_xhigh; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: DFWQcrbMegFAcEWZKWoJxC. Started at 2026-10-13T13:54:48.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.63 and stderr=0.048523658709390974. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":61.111111111111114,"measurements":[{"observationId":"research:gpt-5.5:38","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.63,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.111111111111114,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_xhigh. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: TijCafTTJUFrJ6jPGvBbgq. Startzeit: 2026-08-27T19:28:25.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.63 und stderr=0.015275252316519465. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: gpt-5.5_xhigh; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: TijCafTTJUFrJ6jPGvBbgq. Started at 2026-08-27T19:28:25.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.63 and stderr=0.015275252316519465. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":75.25252525252526,"measurements":[{"observationId":"research:gpt-5.5:32","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.725,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":77.77777777777777,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_xhigh. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: PKSMspJiJdNThB6qNvaBSM. Startzeit: 2026-06-11T17:47:36.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.725 und stderr=0.07149950690165274. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: gpt-5.5_xhigh; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: PKSMspJiJdNThB6qNvaBSM. Started at 2026-06-11T17:47:36.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.725 and stderr=0.07149950690165274. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gpt-5.5:31","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.8526315789473684,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":72.72727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_xhigh. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: 5FGXKvXQwGPtcCX2PNrLR5. Startzeit: 2026-06-11T17:47:36.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.8526315789473684 und stderr=0.021034091168852485. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: gpt-5.5_xhigh; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: 5FGXKvXQwGPtcCX2PNrLR5. Started at 2026-06-11T17:47:36.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.8526315789473684 and stderr=0.021034091168852485. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":43.75,"measurements":[{"observationId":"research:gpt-5.5:34","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.7727272727272727,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":0,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-5.5_none. Epoch-Modellbezeichnung: GPT-5.5. Anzeigenamenfeld: GPT-5.5 (no thinking). Im Alias kodierte Reasoning-Stufe: none. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: KesKa5EXtFG7H4UMBFzgCh. Startzeit: 2026-08-07T00:57:11.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.7727272727272727 und stderr=0.029857515673386438. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: none.","en":"Independent evaluation. Epoch run alias: gpt-5.5_none; Epoch model label: GPT-5.5; Display label field: GPT-5.5 (no thinking); Reasoning setting encoded in the alias: none. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: KesKa5EXtFG7H4UMBFzgCh. Started at 2026-08-07T00:57:11.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.7727272727272727 and stderr=0.029857515673386438. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: none."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: none. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: none. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"none","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.182,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.5,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 112.326. Standardfehler: 1.288. Kosten je Aufgabe in USD: 0.082767. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 112.326. Standard error: 1.288. Cost per task in USD: 0.082767. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-gpt-5.5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":62.8,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":50,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-5209fe0c6c3c. Prüfsumme der Quelldaten: 5209fe0c6c3c30d55e8ab6045e6424f51001495ce27181b7698f6ecce2a6ec5e. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-5209fe0c6c3c. Source payload hash: 5209fe0c6c3c30d55e8ab6045e6424f51001495ce27181b7698f6ecce2a6ec5e. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-gpt-5.5","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":79.5,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":50,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-b34eddc3b10f. Prüfsumme der Quelldaten: b34eddc3b10f43023730e1641a36254635b3dfaca01c5e40f405643dd9ad4a49. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-b34eddc3b10f. Source payload hash: b34eddc3b10f43023730e1641a36254635b3dfaca01c5e40f405643dd9ad4a49. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":45.783132530120504,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 45,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 45.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"lmarena:search:overall:gpt-5.5-search","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1246.55450092229,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":92.3076923076923,"participants":14,"methodology":{"de":"Arena-Bewertung aus 125.957 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.5-search. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.5-search. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 125,957 published pairwise votes.. Source model ID: lmarena/gpt-5.5-search. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: gpt-5.5-search. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 125.957 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 125,957 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 125.957 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 125,957 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":77.73556231003039,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":79.8843930635838,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.42857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 79,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 79.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.266,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.04255319148936,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 54.147. Standardfehler: 0.774. Kosten je Aufgabe in USD: 0.043319. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 54.147. Standard error: 0.774. Cost per task in USD: 0.043319. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":76.27118644067797,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":82.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.27118644067797,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 426.425. Standardfehler: 1.695. Kosten je Aufgabe in USD: 1.361765. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 426.425. Standard error: 1.695. Cost per task in USD: 1.361765. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":80.3658536585366,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:1f054429-397e-4fdb-9e71-67bc92c1735e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":14.6464646464646,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 14,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/1f054429-397e-4fdb-9e71-67bc92c1735e/gpt-5-5/gpt-5-5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Model snapshot: GPT-5.5 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 14.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.5 (Xhigh) (Release gpt-5-5, Zeile gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: GPT-5.5 (Xhigh) (release gpt-5-5, row gpt-5-5, UUID 1f054429-397e-4fdb-9e71-67bc92c1735e, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":73.202,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":43,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2298.224. Standardfehler: 3.978. Kosten je Aufgabe in USD: 1.819086. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 2298.224. Standard error: 3.978. Cost per task in USD: 1.819086. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":76.404,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 427.412. Standardfehler: 0.649. Kosten je Aufgabe in USD: 0.741883. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Maximum output tokens: 128000. Latency in seconds: 427.412. Standard error: 0.649. Cost per task in USD: 0.741883. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 128000","en":"Reasoning effort: high. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":77.41935483870968,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":45.161,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.41935483870968,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1894.953. Standardfehler: 4.164. Kosten je Aufgabe in USD: 6.442253. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1894.953. Standard error: 4.164. Cost per task in USD: 6.442253. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":91.89189189189189,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":68.415,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.89189189189189,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 23.304. Standardfehler: 0.916. Kosten je Aufgabe in USD: 0.43166. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 23.304. Standard error: 0.916. Cost per task in USD: 0.43166. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":64.40677966101696,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":64.543,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.40677966101696,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 910.575. Standardfehler: 2.869. Kosten je Aufgabe in USD: 3.273352. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 910.575. Standard error: 2.869. Cost per task in USD: 3.273352. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":55,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":51.76,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 662.269. Standardfehler: 0.55. Kosten je Aufgabe in USD: 4.149702. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 662.269. Standard error: 0.55. Cost per task in USD: 4.149702. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":68.83116883116882,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.296,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.83116883116882,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 166.845. Standardfehler: 1.018. Kosten je Aufgabe in USD: 0.0666. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 166.845. Standard error: 1.018. Cost per task in USD: 0.0666. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":55.73770491803279,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":3.75,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.73770491803279,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 733.847. Standardfehler: 1.174. Kosten je Aufgabe in USD: 4.597894. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 733.847. Standard error: 1.174. Cost per task in USD: 4.597894. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":87.17948717948718,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.17948717948718,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.139. Standardfehler: 0.41. Kosten je Aufgabe in USD: 0.015403. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 18.139. Standard error: 0.41. Cost per task in USD: 0.015403. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":40.385,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2265.105. Standardfehler: 3.41. Kosten je Aufgabe in USD: 7.402471. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2265.105. Standard error: 3.41. Cost per task in USD: 7.402471. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":73.13432835820896,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.1,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.13432835820896,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 159.546. Standardfehler: 2.188. Kosten je Aufgabe in USD: 0.160759. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 159.546. Standard error: 2.188. Cost per task in USD: 0.160759. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":79.41176470588235,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":86.868,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.41176470588235,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 132.761. Standardfehler: 1.932. Kosten je Aufgabe in USD: 0.142988. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 132.761. Standard error: 1.932. Cost per task in USD: 0.142988. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":79.22077922077922,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":88.144,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.22077922077922,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 42.114. Standardfehler: 0.318. Kosten je Aufgabe in USD: 0.027891. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 42.114. Standard error: 0.318. Cost per task in USD: 0.027891. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.76,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 28.243. Standardfehler: 0.912. Kosten je Aufgabe in USD: 0.071407. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 28.243. Standard error: 0.912. Cost per task in USD: 0.071407. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1322.959. Standardfehler: 0.5. Kosten je Aufgabe in USD: 6.95454. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1322.959. Standard error: 0.5. Cost per task in USD: 6.95454. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":84.48275862068965,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":51.532,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.48275862068965,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 76.05. Standardfehler: 3.954. Kosten je Aufgabe in USD: 0.136289. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 76.05. Standard error: 3.954. Cost per task in USD: 0.136289. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":89.65517241379311,"measurements":[{"observationId":"vals-skillsbench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":62.209,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.65517241379311,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 414.615. Standardfehler: 4.523. Kosten je Aufgabe in USD: 2.54346. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 414.615. Standard error: 4.523. Cost per task in USD: 2.54346. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":36.8421052631579,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":60.893,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.8421052631579,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2377.308. Standardfehler: 1.269. Kosten je Aufgabe in USD: 3.971165. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2377.308. Standard error: 1.269. Cost per task in USD: 3.971165. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":39.583333333333336,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":3.817,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.583333333333336,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2787.997. Standardfehler: 1.186. Kosten je Aufgabe in USD: 17.773692. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 2787.997. Standard error: 1.186. Cost per task in USD: 17.773692. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":47.27272727272727,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":21.762,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.27272727272727,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 951.811. Standardfehler: 2.978. Kosten je Aufgabe in USD: 4.073277. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 951.811. Standard error: 2.978. Cost per task in USD: 4.073277. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":77.63157894736842,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.98,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.63157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 95.83. Standardfehler: 0.854. Kosten je Aufgabe in USD: 0.06042. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 95.83. Standard error: 0.854. Cost per task in USD: 0.06042. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":62.5,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":69.847,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.5. Name des Quellmodells: GPT-5.5. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1911.567. Standardfehler: 4.535. Kosten je Aufgabe in USD: 16.661911. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.5. Source model name: GPT-5.5. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 1911.567. Standard error: 4.535. Cost per task in USD: 16.661911. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":71.36532494586935,"coveredFamilies":42,"totalFamilies":71,"sourceCount":5,"coverage":0.5765873015873016,"lowerBound":41.14834013743975,"upperBound":83.48960997870958,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.3555555555555556,"lowerBound":24.93744954922119,"upperBound":89.38189399366563,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":77.82133178474642,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":50.02799900447984,"upperBound":85.74228471876556,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":15.959595959595958,"upperBound":75.95959595959596,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":72.7935569398984,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":72.7935569398984,"upperBound":72.7935569398984,"provisional":true,"configurationsDiffer":true},"agentic":{"score":67.99299768690234,"coveredFamilies":9,"totalFamilies":12,"sourceCount":3,"coverage":0.75,"lowerBound":50.99474826517675,"upperBound":75.99474826517675,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":25.597826086956523,"upperBound":85.59782608695653,"provisional":true,"configurationsDiffer":true},"finance":{"score":70.1363937313982,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":60.116908912627025,"upperBound":74.40262319834132,"provisional":true,"configurationsDiffer":true},"legal":{"score":69.8612862547289,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":52.395964691046665,"upperBound":77.39596469104666,"provisional":true,"configurationsDiffer":true},"health":{"score":76.27304653204565,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":50.848697688030434,"upperBound":84.18203102136377,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":19.433890577507597,"upperBound":94.4338905775076,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":92.3076923076923,"upperBound":92.3076923076923,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":18.42105263157895,"upperBound":68.42105263157895,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":42.241379310344826,"upperBound":92.24137931034483,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.5-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"gpt-5.6-luna","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":74.39024390243902,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1342.18,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.39024390243902,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":80.76923076923077,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":35.8407079646018,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.76923076923077,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 35,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 35.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":50.208617637561936,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 50,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 50.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":24,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 24,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 24.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1459.2,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":54.285714285714285,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":40.3201506591337,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.285714285714285,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 40,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 40.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":-10.283333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":78.28947368421052,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":91.666,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.28947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 53.521. Standardfehler: 1.736. Kosten je Aufgabe in USD: 0.010616. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 53.521. Standard error: 1.736. Cost per task in USD: 0.010616. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":63.41463414634146,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":39.4810009267841,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.41463414634146,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 39,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 39.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":19.444444444444443,"measurements":[{"observationId":"lmarena:overall:gpt-5.6-luna-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1452.509427905681,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":19.444444444444443,"participants":37,"methodology":{"de":"Arena-Bewertung aus 41.103 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-luna-xhigh. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-luna-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 41,103 published pairwise votes.. Source model ID: lmarena/gpt-5.6-luna-xhigh. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-luna-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 41.103 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 41,103 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 41.103 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 41,103 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":23.333333333333332,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-5.6-luna-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1257.6616631426245,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":23.333333333333332,"participants":31,"methodology":{"de":"Arena-Bewertung aus 11.086 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-luna-xhigh. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-luna-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,086 published pairwise votes.. Source model ID: lmarena/gpt-5.6-luna-xhigh. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-luna-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.086 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 11,086 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.086 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 11,086 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":61.483662613981764,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":78.5549132947977,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.392857142857146,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 78,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 78.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":85.029,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.57446808510638,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 35.102. Standardfehler: 0.858. Kosten je Aufgabe in USD: 0.006419. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 35.102. Standard error: 0.858. Cost per task in USD: 0.006419. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":88.13559322033899,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":93,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.13559322033899,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 201.075. Standardfehler: 1.142. Kosten je Aufgabe in USD: 0.042712. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 201.075. Standard error: 1.142. Cost per task in USD: 0.042712. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":63.15896035408231,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":11.6161616161616,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 11,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 11.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":79.026,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.14285714285714,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 355.751. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.053767. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 355.751. Standard error: 0.991. Cost per task in USD: 0.053767. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":11.616,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.432432432432435,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2155.227. Standardfehler: 1.01. Kosten je Aufgabe in USD: 0.731646. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2155.227. Standard error: 1.01. Cost per task in USD: 0.731646. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":13.288920056100983,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:426d24c8-49ae-482a-b4a8-20f1c53f21c1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":3.33333333333333,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.73913043478261,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 3,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/426d24c8-49ae-482a-b4a8-20f1c53f21c1/gpt-5-6-luna/gpt-5-6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 3.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Luna (Max) (Release gpt-5-6-luna, Zeile gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Luna (Max) (release gpt-5-6-luna, row gpt-5-6-luna, UUID 426d24c8-49ae-482a-b4a8-20f1c53f21c1, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.838709677419355,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7396.452. Standardfehler: 0. Kosten je Aufgabe in USD: 0.581456. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 7396.452. Standard error: 0. Cost per task in USD: 0.581456. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":13.157894736842104,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":61.481,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.157894736842104,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1111.274. Standardfehler: 0.741. Kosten je Aufgabe in USD: 0.088859. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1111.274. Standard error: 0.741. Cost per task in USD: 0.088859. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":70.96774193548387,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.96774193548387,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3514.758. Standardfehler: 4.244. Kosten je Aufgabe in USD: 1.882918. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3514.758. Standard error: 4.244. Cost per task in USD: 1.882918. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":64.86486486486487,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":64.219,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.86486486486487,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.968. Standardfehler: 0.945. Kosten je Aufgabe in USD: 0.019039. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 16.968. Standard error: 0.945. Cost per task in USD: 0.019039. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":73.631,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.27272727272727,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 975.203. Standardfehler: 5.564. Kosten je Aufgabe in USD: 0.404245. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 975.203. Standard error: 5.564. Cost per task in USD: 0.404245. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":76.27118644067797,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":67.116,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.27118644067797,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 660.047. Standardfehler: 2.936. Kosten je Aufgabe in USD: 0.366502. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 660.047. Standard error: 2.936. Cost per task in USD: 0.366502. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":76.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":55.044,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 772.153. Standardfehler: 0.309. Kosten je Aufgabe in USD: 0.279224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 772.153. Standard error: 0.309. Cost per task in USD: 0.279224. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":57.57575757575758,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":61.778,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.57575757575758,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4175.412. Standardfehler: 11.613. Kosten je Aufgabe in USD: 0.57961. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4175.412. Standard error: 11.613. Cost per task in USD: 0.57961. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":31.147540983606557,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.25,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.147540983606557,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 656.357. Standardfehler: 0.833. Kosten je Aufgabe in USD: 0.383981. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 656.357. Standard error: 0.833. Cost per task in USD: 0.383981. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":64.1025641025641,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.1025641025641,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5.886. Standardfehler: 0.421. Kosten je Aufgabe in USD: 0.000942. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 5.886. Standard error: 0.421. Cost per task in USD: 0.000942. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":55,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":36.538,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2373.859. Standardfehler: 3.347. Kosten je Aufgabe in USD: 0.845008. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2373.859. Standard error: 3.347. Cost per task in USD: 0.845008. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":44.776119402985074,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":42.391,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.776119402985074,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 81.283. Standardfehler: 2.27. Kosten je Aufgabe in USD: 0.01597. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 81.283. Standard error: 2.27. Cost per task in USD: 0.01597. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":63.23529411764706,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":84.391,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.23529411764706,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 116.886. Standardfehler: 2.585. Kosten je Aufgabe in USD: 0.022813. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 116.886. Standard error: 2.585. Cost per task in USD: 0.022813. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":57.142857142857146,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.036,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.142857142857146,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.948. Standardfehler: 0.35. Kosten je Aufgabe in USD: 0.002842. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 16.948. Standard error: 0.35. Cost per task in USD: 0.002842. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":67.29,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 22.835. Standardfehler: 0.921. Kosten je Aufgabe in USD: 0.004804. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 22.835. Standard error: 0.921. Cost per task in USD: 0.004804. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":5,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":14.414,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 493.052. Standardfehler: 2.363. Kosten je Aufgabe in USD: 0.109246. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 493.052. Standard error: 2.363. Cost per task in USD: 0.109246. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2493.897. Standardfehler: 0. Kosten je Aufgabe in USD: 0.942725. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2493.897. Standard error: 0. Cost per task in USD: 0.942725. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":58.97435897435897,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":60,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.97435897435897,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 513.687. Standardfehler: 4.924. Kosten je Aufgabe in USD: 0.123288. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 513.687. Standard error: 4.924. Cost per task in USD: 0.123288. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":39.6551724137931,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":44.22,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.6551724137931,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 51.878. Standardfehler: 3.331. Kosten je Aufgabe in USD: 0.010512. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 51.878. Standard error: 3.331. Cost per task in USD: 0.010512. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":82.75862068965517,"measurements":[{"observationId":"vals-skillsbench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":60.447,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.75862068965517,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 320.466. Standardfehler: 4.75. Kosten je Aufgabe in USD: 0.206587. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 320.466. Standard error: 4.75. Cost per task in USD: 0.206587. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":39.473684210526315,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":61.164,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.473684210526315,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2319.748. Standardfehler: 1.268. Kosten je Aufgabe in USD: 0.270182. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2319.748. Standard error: 1.268. Cost per task in USD: 0.270182. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":29.166666666666668,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":1.527,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.166666666666668,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6114.391. Standardfehler: 0.759. Kosten je Aufgabe in USD: 3.439085. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 6114.391. Standard error: 0.759. Cost per task in USD: 3.439085. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":52.72727272727273,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":24.815,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.72727272727273,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2565.695. Standardfehler: 3.113. Kosten je Aufgabe in USD: 0.465359. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2565.695. Standard error: 3.113. Cost per task in USD: 0.465359. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":96.71052631578948,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":76.166,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.71052631578948,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 76.52. Standardfehler: 0.839. Kosten je Aufgabe in USD: 0.012217. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 76.52. Standard error: 0.839. Cost per task in USD: 0.012217. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"vals-vcb-1-100:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":22.594,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.42857142857143,"participants":22,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7374.465. Standardfehler: 4.003. Kosten je Aufgabe in USD: 4.914059. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 7374.465. Standard error: 4.003. Cost per task in USD: 4.914059. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":69.44444444444444,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":77.055,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.44444444444444,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-luna. Name des Quellmodells: GPT-5.6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1551.831. Standardfehler: 3.069. Kosten je Aufgabe in USD: 0.726426. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-luna. Source model name: GPT-5.6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1551.831. Standard error: 3.069. Cost per task in USD: 0.726426. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":54.85973714954892,"coveredFamilies":41,"totalFamilies":71,"sourceCount":3,"coverage":0.561281179138322,"lowerBound":30.79173795451722,"upperBound":74.66362004068502,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":2,"coverage":0.48888888888888904,"lowerBound":26.9263663287087,"upperBound":78.0374774398198,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":36.4283421969558,"upperBound":93.57119933981295,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.794871794871794,"upperBound":91.7948717948718,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":49.912891986062725,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":33.275261324041814,"upperBound":66.60859465737515,"provisional":true,"configurationsDiffer":true},"agentic":{"score":72.8876268110919,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":36.44381340554595,"upperBound":86.44381340554595,"provisional":true,"configurationsDiffer":true},"general":{"score":16.692251461988306,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":13.353801169590644,"upperBound":33.35380116959064,"provisional":true,"configurationsDiffer":true},"finance":{"score":71.46316309228888,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":61.254139793390465,"upperBound":75.53985407910476,"provisional":true,"configurationsDiffer":true},"legal":{"score":50.08336836205689,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":37.562526271542666,"upperBound":62.562526271542666,"provisional":true,"configurationsDiffer":true},"health":{"score":54.005706760316066,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":36.00380450687738,"upperBound":69.33713784021072,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":77.27272727272727,"upperBound":77.27272727272727,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":42.408497973657546,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":21.204248986828773,"upperBound":71.20424898682877,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":19.736842105263158,"upperBound":69.73684210526315,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":19.82758620689655,"upperBound":69.82758620689656,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.6-sol","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":71.05263157894737,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":47.5,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.05263157894737,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 47,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 47.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1480.04,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":96.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":84,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 84,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 84.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":86.58536585365853,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":60.08114996276329,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 60,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 60.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":27.200000000000003,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 27,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 27.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":86.58536585365853,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1608.63,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":76.81159420289855,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":72.6530612244898,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.81159420289855,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 72,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 72.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":56.2146892655367,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 56,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 56.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":21.966666666666665,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"enigma-eval","area":"reasoning","label":{"de":"EnigmaEval","en":"EnigmaEval"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"curated:scale-enigma-eval-2026-07-23-gpt-5.6-sol","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/enigma_eval","benchmarkId":"enigma-eval","benchmarkVersion":"open-source release 2026-07-23","comparisonCohort":"scale-enigma-eval-2026-07-23","retrievedAt":"2026-08-26","value":37.12,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":75,"participants":5,"methodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Kennung des Quellmodells: curated/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-high. Evaluator: Scale AI. Datensatztyp: public. Stand der Rangliste: 2026-07-23. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 37,12 % ± 2,8. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-889855e1756c. Prüfsumme der Quelldaten: 889855e1756c00809ef4846f75dd5fd218ff0799db4084a9fecbfa747ed6959e. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Source model ID: curated/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-high. Evaluator: Scale AI. Dataset type: public. Leaderboard date: 2026-07-23. Attempts: pass@1. Reasoning effort: high. Score display: 37.12% ± 2.8. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-889855e1756c. Source payload hash: 889855e1756c00809ef4846f75dd5fd218ff0799db4084a9fecbfa747ed6959e. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.","en":"Complex multimodal puzzles with exact answer matching and one attempt."},"scoreRunConfiguration":{"de":"Komplexe multimodale Rätsel mit exaktem Antwortabgleich und einem Versuch.. Versuche: pass@1. Reasoning-Stufe: high","en":"Complex multimodal puzzles with exact answer matching and one attempt.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-enigma-eval-2026-07-23","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":98.6842105263158,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":95.202,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.6842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 58.304. Standardfehler: 1.074. Kosten je Aufgabe in USD: 0.10996. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 58.304. Standard error: 1.074. Cost per task in USD: 0.10996. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":49.4902687673772,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 49,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 49.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":100,"measurements":[{"observationId":"lmarena:search:overall:gpt-5.6-sol-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1261.1307593350664,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":100,"participants":14,"methodology":{"de":"Arena-Bewertung aus 60.437 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-sol-xhigh. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 60,437 published pairwise votes.. Source model ID: lmarena/gpt-5.6-sol-xhigh. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 60.437 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: xhigh","en":"Arena rating from 60,437 published pairwise votes.. Harness: Arena Search. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 60.437 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: xhigh","en":"Arena rating from 60,437 published pairwise votes.. Harness: Arena Search. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":72.22222222222223,"measurements":[{"observationId":"lmarena:overall:gpt-5.6-sol-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1484.859970239758,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":72.22222222222223,"participants":37,"methodology":{"de":"Arena-Bewertung aus 39.832 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-sol-xhigh. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 39,832 published pairwise votes.. Source model ID: lmarena/gpt-5.6-sol-xhigh. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 39.832 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 39,832 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 39.832 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 39,832 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":76.66666666666667,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-5.6-sol-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1286.9619006302116,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":76.66666666666667,"participants":31,"methodology":{"de":"Arena-Bewertung aus 10.907 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-sol-xhigh. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,907 published pairwise votes.. Source model ID: lmarena/gpt-5.6-sol-xhigh. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.907 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 10,907 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.907 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 10,907 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":89.32370820668693,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83.4104046242775,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.28571428571429,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.844,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.36170212765957,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 26.198. Standardfehler: 0.757. Kosten je Aufgabe in USD: 0.055795. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 26.198. Standard error: 0.757. Cost per task in USD: 0.055795. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"curated:scale-prbench-finance-gpt-5.6-sol","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":50.45,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":73.33333333333333,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-max. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 50,45 % ± 0,33. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-d629f8c083e2. Prüfsumme der Quelldaten: d629f8c083e2a0fcb03c088191f3ab1430e7b2a4e8f54278c34edc74edda4cb1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-max. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 50.45% ± 0.33. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-d629f8c083e2. Source payload hash: d629f8c083e2a0fcb03c088191f3ab1430e7b2a4e8f54278c34edc74edda4cb1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: max","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"curated:scale-prbench-legal-gpt-5.6-sol","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":50.5,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-sol-max. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: max. Angezeigter Wert: 50,5 % ± 0,54. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-24d55597426f. Prüfsumme der Quelldaten: 24d55597426fd029fa24e102f770e003620b79cd61d3cb0ceaf589ac14791af7. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-sol-max. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: max. Score display: 50.5% ± 0.54. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-24d55597426f. Source payload hash: 24d55597426fd029fa24e102f770e003620b79cd61d3cb0ceaf589ac14791af7. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1. Reasoning-Stufe: max","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":98.30508474576271,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":96.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.30508474576271,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 182.365. Standardfehler: 0.856. Kosten je Aufgabe in USD: 1.1513. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 182.365. Standard error: 0.856. Cost per task in USD: 1.1513. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":87.48610980318298,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":39.8989898989899,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 39,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 39.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":85.768,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.42857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 372.819. Standardfehler: 1.35. Kosten je Aufgabe in USD: 1.023319. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 372.819. Standard error: 1.35. Cost per task in USD: 1.023319. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":37.879,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.27027027027027,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2026.072. Standardfehler: 0. Kosten je Aufgabe in USD: 7.983597. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2026.072. Standard error: 0. Cost per task in USD: 7.983597. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":74.61430575035064,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:d93edfe8-bf35-49ad-b56e-b18116142a1c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":22.3809523809524,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.26086956521739,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 22,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/d93edfe8-bf35-49ad-b56e-b18116142a1c/gpt-5-6-sol/gpt-5-6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 22.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Sol (Max) (Release gpt-5-6-sol, Zeile gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Sol (Max) (release gpt-5-6-sol, row gpt-5-6-sol, UUID d93edfe8-bf35-49ad-b56e-b18116142a1c, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":20,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.96774193548387,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6216.027. Standardfehler: 4.815. Kosten je Aufgabe in USD: 7.322071. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 6216.027. Standard error: 4.815. Cost per task in USD: 7.322071. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":52.63157894736842,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":71.111,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.63157894736842,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1098.261. Standardfehler: 0.642. Kosten je Aufgabe in USD: 1.925036. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1098.261. Standard error: 0.642. Cost per task in USD: 1.925036. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":83.87096774193549,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":52.916,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.87096774193549,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3551.876. Standardfehler: 4.353. Kosten je Aufgabe in USD: 24.541448. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3551.876. Standard error: 4.353. Cost per task in USD: 24.541448. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":66.21621621621621,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":64.375,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.21621621621621,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15.941. Standardfehler: 0.943. Kosten je Aufgabe in USD: 0.484982. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 15.941. Standard error: 0.943. Cost per task in USD: 0.484982. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":93.93939393939394,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":76.31,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.93939393939394,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 646.763. Standardfehler: 5.177. Kosten je Aufgabe in USD: 3.048302. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 646.763. Standard error: 5.177. Cost per task in USD: 3.048302. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":91.52542372881356,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":72.339,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.52542372881356,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 590.258. Standardfehler: 2.373. Kosten je Aufgabe in USD: 6.010773. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 590.258. Standard error: 2.373. Cost per task in USD: 6.010773. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.756,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1165.258. Standardfehler: 0.846. Kosten je Aufgabe in USD: 1.253419. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1165.258. Standard error: 0.846. Cost per task in USD: 1.253419. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":91.167,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3711.631. Standardfehler: 4.51. Kosten je Aufgabe in USD: 7.982135. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3711.631. Standard error: 4.51. Cost per task in USD: 7.982135. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.604,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 56.513. Standardfehler: 1.088. Kosten je Aufgabe in USD: 0.089831. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 56.513. Standard error: 1.088. Cost per task in USD: 0.089831. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":46.721311475409834,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.5,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.721311475409834,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1370.296. Standardfehler: 0.833. Kosten je Aufgabe in USD: 10.374554. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1370.296. Standard error: 0.833. Cost per task in USD: 10.374554. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":91.02564102564102,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.965,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.02564102564102,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6.198. Standardfehler: 0.412. Kosten je Aufgabe in USD: 0.012678. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 6.198. Standard error: 0.412. Cost per task in USD: 0.012678. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":48.077,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4631.383. Standardfehler: 3.473. Kosten je Aufgabe in USD: 21.605566. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4631.383. Standard error: 3.473. Cost per task in USD: 21.605566. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":53.73134328358209,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":43.974,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.73134328358209,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 96.589. Standardfehler: 2.258. Kosten je Aufgabe in USD: 0.280517. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 96.589. Standard error: 2.258. Cost per task in USD: 0.280517. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":69.11764705882354,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":85.233,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.11764705882354,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 94.404. Standardfehler: 1.973. Kosten je Aufgabe in USD: 0.276691. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 94.404. Standard error: 1.973. Cost per task in USD: 0.276691. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":83.11688311688312,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.1,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.11688311688312,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15.654. Standardfehler: 0.308. Kosten je Aufgabe in USD: 0.032087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 15.654. Standard error: 0.308. Cost per task in USD: 0.032087. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":59.61538461538461,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":67.289,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.61538461538461,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.618. Standardfehler: 0.922. Kosten je Aufgabe in USD: 0.071198. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 12.618. Standard error: 0.922. Cost per task in USD: 0.071198. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":55,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":33.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 522.744. Standardfehler: 3.171. Kosten je Aufgabe in USD: 0.96335. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 522.744. Standard error: 3.171. Cost per task in USD: 0.96335. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":78.57142857142857,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":1.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.57142857142857,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1764.488. Standardfehler: 0.862. Kosten je Aufgabe in USD: 15.289359. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1764.488. Standard error: 0.862. Cost per task in USD: 15.289359. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":70.51282051282051,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":83,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.51282051282051,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 520.405. Standardfehler: 3.775. Kosten je Aufgabe in USD: 1.438313. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 520.405. Standard error: 3.775. Cost per task in USD: 1.438313. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":93.10344827586206,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":52.563,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.10344827586206,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 75.395. Standardfehler: 3.416. Kosten je Aufgabe in USD: 0.147977. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 75.395. Standard error: 3.416. Cost per task in USD: 0.147977. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":58.62068965517241,"measurements":[{"observationId":"vals-skillsbench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":54.1,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.62068965517241,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 787.486. Standardfehler: 4.73. Kosten je Aufgabe in USD: 4.13999. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 787.486. Standard error: 4.73. Cost per task in USD: 4.13999. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":68.42105263157895,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":66.509,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.42105263157895,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3677.634. Standardfehler: 1.228. Kosten je Aufgabe in USD: 8.847882. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3677.634. Standard error: 1.228. Cost per task in USD: 8.847882. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":85.41666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":30.534,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.41666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5285.426. Standardfehler: 2.851. Kosten je Aufgabe in USD: 42.521152. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 5285.426. Standard error: 2.851. Cost per task in USD: 42.521152. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":70.9090909090909,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":31.088,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.9090909090909,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2681.175. Standardfehler: 3.34. Kosten je Aufgabe in USD: 7.513965. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2681.175. Standard error: 3.34. Cost per task in USD: 7.513965. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":73.6842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.775,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.6842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 62.034. Standardfehler: 0.856. Kosten je Aufgabe in USD: 0.138418. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 62.034. Standard error: 0.856. Cost per task in USD: 0.138418. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":61.904761904761905,"measurements":[{"observationId":"vals-vcb-1-100:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":20.011,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.904761904761905,"participants":22,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 9421.289. Standardfehler: 3.629. Kosten je Aufgabe in USD: 47.425154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 9421.289. Standard error: 3.629. Cost per task in USD: 47.425154. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":75,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":80.495,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-sol. Name des Quellmodells: GPT-5.6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2033.047. Standardfehler: 3.716. Kosten je Aufgabe in USD: 33.403634. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-sol. Source model name: GPT-5.6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2033.047. Standard error: 3.716. Cost per task in USD: 33.403634. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":80.61311089427373,"coveredFamilies":47,"totalFamilies":71,"sourceCount":4,"coverage":0.6900226757369613,"lowerBound":55.624874478747145,"upperBound":86.62260690505101,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":76.97683719582672,"coveredFamilies":9,"totalFamilies":15,"sourceCount":2,"coverage":0.5555555555555557,"lowerBound":42.764909553237075,"upperBound":87.2093539976815,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":90.68918485237484,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.5714285714285714,"lowerBound":51.822391344214196,"upperBound":94.67953420135706,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":14.102564102564102,"upperBound":94.1025641025641,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":81.80234399746594,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":54.53489599831063,"upperBound":87.86822933164396,"provisional":true,"configurationsDiffer":true},"agentic":{"score":80.9618137016216,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":40.4809068508108,"upperBound":90.4809068508108,"provisional":true,"configurationsDiffer":true},"general":{"score":68.41641240783119,"coveredFamilies":5,"totalFamilies":5,"sourceCount":3,"coverage":1,"lowerBound":68.41641240783119,"upperBound":68.41641240783119,"provisional":true,"configurationsDiffer":true},"finance":{"score":71.70718942797444,"coveredFamilies":7,"totalFamilies":7,"sourceCount":2,"coverage":1,"lowerBound":71.70718942797444,"upperBound":71.70718942797444,"provisional":true,"configurationsDiffer":true},"legal":{"score":77.77007145859605,"coveredFamilies":4,"totalFamilies":4,"sourceCount":2,"coverage":1,"lowerBound":77.77007145859605,"upperBound":77.77007145859605,"provisional":true,"configurationsDiffer":true},"health":{"score":61.424495171202814,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":40.949663447468545,"upperBound":74.28299678080188,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":93.93939393939394,"upperBound":93.93939393939394,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":82.9951874366768,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":41.4975937183384,"upperBound":91.49759371833841,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":100,"upperBound":100,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":34.21052631578947,"upperBound":84.21052631578948,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":46.55172413793103,"upperBound":96.55172413793103,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-5.6-terra","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1334.22,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":88.41463414634147,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.41463414634147,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":88.46153846153847,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":38.9380530973451,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.46153846153847,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 38,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 38.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":59.64995802534495,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 59,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 59.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":24,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 24,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 24.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":74.39024390243902,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1451.58,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.39024390243902,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":71.01449275362319,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":71.2244897959184,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.01449275362319,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 71,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 71.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":91.42857142857143,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":51.035781544256096,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.42857142857143,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 51,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 51.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":51.21951219512195,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":0.05,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.21951219512195,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":75,"measurements":[{"observationId":"vals-gpqa:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":90.909,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 36.764. Standardfehler: 1.906. Kosten je Aufgabe in USD: 0.033997. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 36.764. Standard error: 1.906. Cost per task in USD: 0.033997. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":42.9101019462465,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 42,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 42.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":41.666666666666664,"measurements":[{"observationId":"lmarena:overall:gpt-5.6-terra-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1465.8969474060723,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":41.666666666666664,"participants":37,"methodology":{"de":"Arena-Bewertung aus 42.358 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-terra-xhigh. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-terra-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 42,358 published pairwise votes.. Source model ID: lmarena/gpt-5.6-terra-xhigh. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-terra-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 42.358 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 42,358 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 42.358 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 42,358 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":46.666666666666664,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-5.6-terra-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1268.0435522963623,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":46.666666666666664,"participants":31,"methodology":{"de":"Arena-Bewertung aus 11.020 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-5.6-terra-xhigh. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-5.6-terra-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,020 published pairwise votes.. Source model ID: lmarena/gpt-5.6-terra-xhigh. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: gpt-5.6-terra-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.020 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 11,020 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.020 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 11,020 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":73.32826747720364,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":80.6936416184971,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.57142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":86.474,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.08510638297872,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 13.033. Standardfehler: 0.822. Kosten je Aufgabe in USD: 0.016375. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 13.033. Standard error: 0.822. Cost per task in USD: 0.016375. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":94.0677966101695,"measurements":[{"observationId":"vals-swebench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":95.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.0677966101695,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 180.284. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.400788. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 180.284. Standard error: 0.938. Cost per task in USD: 0.400788. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":78.48891138525286,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":35.3535353535354,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 35,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 35.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":77.528,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.67857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 519.739. Standardfehler: 2.247. Kosten je Aufgabe in USD: 0.472552. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 519.739. Standard error: 2.247. Cost per task in USD: 0.472552. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":22.727,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.351351351351354,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1888.568. Standardfehler: 2.314. Kosten je Aufgabe in USD: 5.602118. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1888.568. Standard error: 2.314. Cost per task in USD: 5.602118. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":62.20196353436185,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:bcf8db0a-3bb6-4d82-9516-0f57370c85a6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":13.8095238095238,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.56521739130434,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 13,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/bcf8db0a-3bb6-4d82-9516-0f57370c85a6/gpt-5-6-terra/gpt-5-6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Model snapshot: GPT-5.6 Terra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 13.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-5.6 Terra (Max) (Release gpt-5-6-terra, Zeile gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-5.6 Terra (Max) (release gpt-5-6-terra, row gpt-5-6-terra, UUID bcf8db0a-3bb6-4d82-9516-0f57370c85a6, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":10,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.83870967741935,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7443.23. Standardfehler: 3.612. Kosten je Aufgabe in USD: 5.19351. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 7443.23. Standard error: 3.612. Cost per task in USD: 5.19351. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":82.25806451612904,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":47.804,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.25806451612904,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2914.132. Standardfehler: 4.28. Kosten je Aufgabe in USD: 8.127254. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2914.132. Standard error: 4.28. Cost per task in USD: 8.127254. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":73.64864864864865,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":65.307,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.64864864864865,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7.717. Standardfehler: 0.938. Kosten je Aufgabe in USD: 0.171459. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 7.717. Standard error: 0.938. Cost per task in USD: 0.171459. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":71.21212121212122,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":72.084,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.21212121212122,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1023.932. Standardfehler: 5.408. Kosten je Aufgabe in USD: 3.308232. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1023.932. Standard error: 5.408. Cost per task in USD: 3.308232. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":67.79661016949153,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":66.205,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.79661016949153,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 813.024. Standardfehler: 3.11. Kosten je Aufgabe in USD: 2.193037. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 813.024. Standard error: 3.11. Cost per task in USD: 2.193037. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":54.436,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1449.852. Standardfehler: 2.071. Kosten je Aufgabe in USD: 3.629197. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1449.852. Standard error: 2.071. Cost per task in USD: 3.629197. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":84.84848484848484,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":87.611,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.84848484848484,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4992.838. Standardfehler: 6.531. Kosten je Aufgabe in USD: 8.664009. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4992.838. Standard error: 6.531. Cost per task in USD: 8.664009. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":74.02597402597402,"measurements":[{"observationId":"vals-lcb:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.93,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.02597402597402,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 44.331. Standardfehler: 1.017. Kosten je Aufgabe in USD: 0.031103. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 44.331. Standard error: 1.017. Cost per task in USD: 0.031103. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":27.868852459016395,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0.833,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.868852459016395,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1024.886. Standardfehler: 0.833. Kosten je Aufgabe in USD: 3.202399. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1024.886. Standard error: 0.833. Cost per task in USD: 3.202399. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"vals-legal-bench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":85.105,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2.782. Standardfehler: 0.45. Kosten je Aufgabe in USD: 0.002643. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2.782. Standard error: 0.45. Cost per task in USD: 0.002643. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":70.83333333333333,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":41.346,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.83333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4172.869. Standardfehler: 3.423. Kosten je Aufgabe in USD: 7.907299. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4172.869. Standard error: 3.423. Cost per task in USD: 7.907299. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":50.74626865671642,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":43.414,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.74626865671642,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 18.411. Standardfehler: 2.173. Kosten je Aufgabe in USD: 0.046558. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 18.411. Standard error: 2.173. Cost per task in USD: 0.046558. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":48.529411764705884,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":82.869,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.529411764705884,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 35.526. Standardfehler: 1.948. Kosten je Aufgabe in USD: 0.062588. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Maximum output tokens: 30000. Latency in seconds: 35.526. Standard error: 1.948. Cost per task in USD: 0.062588. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":64.93506493506493,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.659,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.93506493506493,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8.173. Standardfehler: 0.333. Kosten je Aufgabe in USD: 0.008039. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 8.173. Standard error: 0.333. Cost per task in USD: 0.008039. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":63.46153846153846,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":67.329,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.46153846153846,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6.316. Standardfehler: 0.925. Kosten je Aufgabe in USD: 0.022706. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 6.316. Standard error: 0.925. Cost per task in USD: 0.022706. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2016.05. Standardfehler: 0.5. Kosten je Aufgabe in USD: 5.377882. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2016.05. Standard error: 0.5. Cost per task in USD: 5.377882. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":64.1025641025641,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":74,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.1025641025641,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 777.341. Standardfehler: 4.408. Kosten je Aufgabe in USD: 1.274209. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 777.341. Standard error: 4.408. Cost per task in USD: 1.274209. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":55.172413793103445,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":47.003,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.172413793103445,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 40.057. Standardfehler: 3.403. Kosten je Aufgabe in USD: 0.037793. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 40.057. Standard error: 3.403. Cost per task in USD: 0.037793. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":72.41379310344827,"measurements":[{"observationId":"vals-skillsbench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":58.895,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.41379310344827,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 415.068. Standardfehler: 4.472. Kosten je Aufgabe in USD: 1.75247. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 415.068. Standard error: 4.472. Cost per task in USD: 1.75247. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":44.73684210526316,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":62.382,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.73684210526316,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1171.486. Standardfehler: 1.26. Kosten je Aufgabe in USD: 1.199723. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1171.486. Standard error: 1.26. Cost per task in USD: 1.199723. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":58.333333333333336,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":9.542,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.333333333333336,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 11800.798. Standardfehler: 1.819. Kosten je Aufgabe in USD: 30.855656. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 11800.798. Standard error: 1.819. Cost per task in USD: 30.855656. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":65.45454545454545,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":30.035,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.45454545454545,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3630.72. Standardfehler: 3.307. Kosten je Aufgabe in USD: 4.829225. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3630.72. Standard error: 3.307. Cost per task in USD: 4.829225. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":96.71052631578948,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":76.166,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.71052631578948,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 37.28. Standardfehler: 0.836. Kosten je Aufgabe in USD: 0.035909. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Maximum output tokens: 128000. Latency in seconds: 37.28. Standard error: 0.836. Cost per task in USD: 0.035909. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Maximale Ausgabetoken: 128000","en":"Reasoning effort: xhigh. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-vcb-1-100:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":14.822,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":22,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6536.122. Standardfehler: 3.123. Kosten je Aufgabe in USD: 33.326696. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 6536.122. Standard error: 3.123. Cost per task in USD: 33.326696. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-5.6-terra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":74.594,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-5.6-terra. Name des Quellmodells: GPT-5.6 Terra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1152.433. Standardfehler: 4.203. Kosten je Aufgabe in USD: 7.887015. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-5.6-terra. Source model name: GPT-5.6 Terra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1152.433. Standard error: 4.203. Cost per task in USD: 7.887015. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":63.65603320389721,"coveredFamilies":41,"totalFamilies":71,"sourceCount":3,"coverage":0.5517573696145125,"lowerBound":35.12268544067639,"upperBound":79.94694847922514,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":69.1175871620478,"coveredFamilies":9,"totalFamilies":15,"sourceCount":2,"coverage":0.5555555555555557,"lowerBound":38.39865953447101,"upperBound":82.84310397891544,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":36.367595818815325,"upperBound":93.51045296167246,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":12.820512820512821,"upperBound":92.82051282051282,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":58.07728856509345,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":38.718192376728965,"upperBound":72.0515257100623,"provisional":true,"configurationsDiffer":true},"agentic":{"score":80.26203261274749,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":40.131016306373745,"upperBound":90.13101630637374,"provisional":true,"configurationsDiffer":true},"general":{"score":57.0048309178744,"coveredFamilies":3,"totalFamilies":5,"sourceCount":3,"coverage":0.6000000000000001,"lowerBound":34.20289855072465,"upperBound":74.20289855072464,"provisional":true,"configurationsDiffer":true},"finance":{"score":73.40086706389116,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":62.9150289119067,"upperBound":77.200743197621,"provisional":true,"configurationsDiffer":true},"legal":{"score":58.54175423847554,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":43.90631567885666,"upperBound":68.90631567885666,"provisional":true,"configurationsDiffer":true},"health":{"score":49.63784021071115,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":33.09189347380743,"upperBound":66.42522680714077,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":71.21212121212122,"upperBound":71.21212121212122,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":59.997467071935155,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":29.998733535967578,"upperBound":79.99873353596757,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":22.36842105263158,"upperBound":72.36842105263158,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":27.586206896551722,"upperBound":77.58620689655172,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-6-astra","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"research:gpt-6-astra:49","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":1,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: L7kW8yxFKCjVXhxNnt3VTq. Startzeit: 2026-08-30T14:57:43.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=1.0 und stderr=0.0. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: L7kW8yxFKCjVXhxNnt3VTq. Started at 2026-08-30T14:57:43.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=1.0 and stderr=0.0. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":81.57894736842105,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":51.24999999999999,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.57894736842105,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 51,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 51.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1568.99,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":72.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":80.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":72.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":97.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":68.49174806688876,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 68,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 68.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":46.9827666666667,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.66666666666667,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 47,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 47.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":98.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":31,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 31,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 31.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1572.84,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":82.85714285714286,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":48.5875706214689,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.85714285714286,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 48,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 48.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":43.4,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"research:gpt-6-astra:47","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.74,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":100,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: KaEir82ukWPm2Fq4FanY2M. Startzeit: 2026-10-13T13:42:38.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.74 und stderr=0.0440844002276808. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: KaEir82ukWPm2Fq4FanY2M. Started at 2026-10-13T13:42:38.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.74 and stderr=0.0440844002276808. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":94.44444444444444,"measurements":[{"observationId":"research:gpt-6-astra:48","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.756,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":94.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: PppwnX7EjXx2SeTcFN8Ujn. Startzeit: 2026-08-30T14:57:43.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.756 und stderr=0.013588548437881386. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: PppwnX7EjXx2SeTcFN8Ujn. Started at 2026-08-30T14:57:43.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.756 and stderr=0.013588548437881386. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":98.86363636363637,"measurements":[{"observationId":"research:gpt-6-astra:42","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.976,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":100,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: f7BCT27LzcAtaLDP2tcrWF. Startzeit: 2026-08-30T02:22:12.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.976 und stderr=0.024. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: f7BCT27LzcAtaLDP2tcrWF. Started at 2026-08-30T02:22:12.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.976 and stderr=0.024. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gpt-6-astra:39","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.9368421052631579,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":97.72727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: csK2fe5T8p2R7WgoJUu6bZ. Startzeit: 2026-08-30T14:57:43.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.9368421052631579 und stderr=0.014434038295343787. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: csK2fe5T8p2R7WgoJUu6bZ. Started at 2026-08-30T14:57:43.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.9368421052631579 and stderr=0.014434038295343787. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":95.23809523809524,"measurements":[{"observationId":"research:gpt-6-astra:46","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9577020202020202,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":95.23809523809524,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-astra_max. Epoch-Modellbezeichnung: GPT-6 Astra. Anzeigenamenfeld: GPT-6 Astra (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: 3h8STJpG2VzY9LiBUzwotv. Startzeit: 2026-08-30T14:57:43.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9577020202020202 und stderr=0.013696812097020311. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-astra_max; Epoch model label: GPT-6 Astra; Display label field: GPT-6 Astra (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: 3h8STJpG2VzY9LiBUzwotv. Started at 2026-08-30T14:57:43.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9577020202020202 and stderr=0.013696812097020311. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":97.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":54.6802594995366,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 54,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 54.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":97.22222222222223,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-6-astra-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1780.244235647691,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":97.22222222222223,"participants":37,"methodology":{"de":"Arena-Bewertung aus 5.746 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-astra-max. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-astra-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 5,746 published pairwise votes.. Source model ID: lmarena/gpt-6-astra-max. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: gpt-6-astra-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 5.746 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 5,746 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 5.746 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 5,746 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"lmarena:search:overall:gpt-6-astra-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1233.6158108189236,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":76.92307692307692,"participants":14,"methodology":{"de":"Arena-Bewertung aus 18.483 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-astra-max. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-astra-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 18,483 published pairwise votes.. Source model ID: lmarena/gpt-6-astra-max. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: gpt-6-astra-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 18.483 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 18,483 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 18.483 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 18,483 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":58.333333333333336,"measurements":[{"observationId":"lmarena:overall:gpt-6-astra-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1476.715634979153,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":58.333333333333336,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.674 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-astra-max. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-astra-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,674 published pairwise votes.. Source model ID: lmarena/gpt-6-astra-max. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: gpt-6-astra-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.674 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 11,674 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.674 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 11,674 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":70,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-6-astra-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1285.9881756419982,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":70,"participants":31,"methodology":{"de":"Arena-Bewertung aus 3.590 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-astra-max. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-astra-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 3,590 published pairwise votes.. Source model ID: lmarena/gpt-6-astra-max. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: gpt-6-astra-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 3.590 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 3,590 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 3.590 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 3,590 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":86.878612716763,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 86,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 86.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":98.56177606177607,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":59.09090909090911,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 59,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 59.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":87.266,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.21428571428571,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 274.44. Standardfehler: 0.375. Kosten je Aufgabe in USD: 1.339493. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 274.44. Standard error: 0.375. Cost per task in USD: 1.339493. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":59.596,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.5945945945946,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2142.082. Standardfehler: 4.403. Kosten je Aufgabe in USD: 9.584259. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2142.082. Standard error: 4.403. Cost per task in USD: 9.584259. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:2f339a97-9a0d-499a-9cb5-e0db665bfa25","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":63.3333333333333,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 63,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/2f339a97-9a0d-499a-9cb5-e0db665bfa25/gpt-6-astra/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Model snapshot: GPT-6 Astra (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 63.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Astra (Max) (Release gpt-6-astra, Zeile gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Astra (Max) (release gpt-6-astra, row gpt-6-astra, UUID 2f339a97-9a0d-499a-9cb5-e0db665bfa25, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":62.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6534.868. Standardfehler: 5.817. Kosten je Aufgabe in USD: 20.802802. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 6534.868. Standard error: 5.817. Cost per task in USD: 20.802802. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":84.21052631578948,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":79.259,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.21052631578948,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 286.147. Standardfehler: 0.98. Kosten je Aufgabe in USD: 1.719326. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 286.147. Standard error: 0.98. Cost per task in USD: 1.719326. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":98.38709677419355,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":67.74,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.38709677419355,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3510.003. Standardfehler: 4.216. Kosten je Aufgabe in USD: 44.364548. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3510.003. Standard error: 4.216. Cost per task in USD: 44.364548. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":41.072,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 250.415. Standardfehler: 2.559. Kosten je Aufgabe in USD: 1.840737. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 250.415. Standard error: 2.559. Cost per task in USD: 1.840737. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":86.44067796610169,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":71.7,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.44067796610169,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 804.28. Standardfehler: 2.475. Kosten je Aufgabe in USD: 5.827063. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 804.28. Standard error: 2.475. Cost per task in USD: 5.827063. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":63.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.54,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 692.672. Standardfehler: 2.078. Kosten je Aufgabe in USD: 6.81746. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 692.672. Standard error: 2.078. Cost per task in USD: 6.81746. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":100,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1899.783. Standardfehler: 0. Kosten je Aufgabe in USD: 6.495051. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1899.783. Standard error: 0. Cost per task in USD: 6.495051. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":64.75409836065573,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":5.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.75409836065573,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1489.086. Standardfehler: 1.174. Kosten je Aufgabe in USD: 26.155899. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1489.086. Standard error: 1.174. Cost per task in USD: 26.155899. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":65,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":39.423,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1491.339. Standardfehler: 3.397. Kosten je Aufgabe in USD: 10.577143. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1491.339. Standard error: 3.397. Cost per task in USD: 10.577143. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":70.14925373134328,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":48.486,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.14925373134328,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 102.865. Standardfehler: 2.131. Kosten je Aufgabe in USD: 0.451358. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 102.865. Standard error: 2.131. Cost per task in USD: 0.451358. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":83.82352941176471,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":87.908,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.82352941176471,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 153.728. Standardfehler: 1.938. Kosten je Aufgabe in USD: 0.581991. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 153.728. Standard error: 1.938. Cost per task in USD: 0.581991. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":53.153,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 488.597. Standardfehler: 3.357. Kosten je Aufgabe in USD: 1.559704. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 488.597. Standard error: 3.357. Cost per task in USD: 1.559704. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":93.87755102040816,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":5.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.87755102040816,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1621.514. Standardfehler: 1.616. Kosten je Aufgabe in USD: 11.574605. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1621.514. Standard error: 1.616. Cost per task in USD: 11.574605. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":89.74358974358974,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":99,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.74358974358974,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 307.867. Standardfehler: 1. Kosten je Aufgabe in USD: 1.71238. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 307.867. Standard error: 1. Cost per task in USD: 1.71238. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":50,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":46.367,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 41.629. Standardfehler: 3.444. Kosten je Aufgabe in USD: 0.282823. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 41.629. Standard error: 3.444. Cost per task in USD: 0.282823. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":56.87,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1070.676. Standardfehler: 3.066. Kosten je Aufgabe in USD: 10.238062. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1070.676. Standard error: 3.066. Cost per task in USD: 10.238062. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"vals-swebench-multimodal:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":43.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.46153846153846,"participants":14,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 423.272. Standardfehler: 2.264. Kosten je Aufgabe in USD: 1.80573. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 423.272. Standard error: 2.264. Cost per task in USD: 1.80573. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":41.81818181818182,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":20.717,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.81818181818182,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 900.586. Standardfehler: 2.924. Kosten je Aufgabe in USD: 5.7963. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 900.586. Standard error: 2.924. Cost per task in USD: 5.7963. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":80.95238095238095,"measurements":[{"observationId":"vals-vcb-1-100:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":27.644,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.95238095238095,"participants":22,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12339.625. Standardfehler: 4.082. Kosten je Aufgabe in USD: 84.720155. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 12339.625. Standard error: 4.082. Cost per task in USD: 84.720155. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":93.05555555555556,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-6-astra","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":89.594,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.05555555555556,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-astra. Name des Quellmodells: GPT-6 Astra. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2344.127. Standardfehler: 2.174. Kosten je Aufgabe in USD: 38.512323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-astra. Source model name: GPT-6 Astra. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2344.127. Standard error: 2.174. Cost per task in USD: 38.512323. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":71.89435439897095,"coveredFamilies":41,"totalFamilies":71,"sourceCount":4,"coverage":0.5843253968253969,"lowerBound":42.009697163684415,"upperBound":83.57715748114472,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":88.55164322418092,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.6555555555555557,"lowerBound":58.05052166918529,"upperBound":92.49496611362972,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":91.78281068524971,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":52.44732039157126,"upperBound":95.30446324871411,"provisional":true,"configurationsDiffer":true},"math":{"score":93.17210567210567,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":55.90326340326341,"upperBound":95.9032634032634,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":97.22222222222223,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":64.81481481481482,"upperBound":98.14814814814815,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":36.26398312855309,"upperBound":94.59731646188642,"provisional":true,"configurationsDiffer":true},"general":{"score":85.63596491228071,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":68.50877192982456,"upperBound":88.50877192982456,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":27.370313302516692,"upperBound":84.51317044537383,"provisional":true,"configurationsDiffer":true},"legal":{"score":64.87704918032787,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":32.43852459016394,"upperBound":82.43852459016394,"provisional":true,"configurationsDiffer":true},"health":{"score":76.986391571554,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":51.32426104770266,"upperBound":84.657594381036,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":9.090909090909092,"upperBound":9.090909090909092,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":30,"upperBound":92.5,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":76.92307692307692,"upperBound":76.92307692307692,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":25,"upperBound":75,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-6-luna","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":61.36363636363637,"measurements":[{"observationId":"research:gpt-6-luna:6","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9888888888888889,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.36363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: 4uDz9b6SosxDSZf7swMD9D. Startzeit: 2026-09-22T22:14:05.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9888888888888889 und stderr=0.006674027922425511. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: 4uDz9b6SosxDSZf7swMD9D. Started at 2026-09-22T22:14:05.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9888888888888889 and stderr=0.006674027922425511. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":53.94736842105263,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":43.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.94736842105263,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 43,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 43.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1334.83,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":70.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":53.16814034483175,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 53,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 53.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":60,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":41.9269666666667,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":22.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 22,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 22.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1431.86,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":52.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":0.65,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":26.19047619047619,"measurements":[{"observationId":"research:gpt-6-luna:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.36,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":26.19047619047619,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: Jevnk9hi9oRBbGcFu2JxBC. Startzeit: 2026-10-13T13:37:06.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.36 und stderr=0.04824181513244218. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: Jevnk9hi9oRBbGcFu2JxBC. Started at 2026-10-13T13:37:06.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.36 and stderr=0.04824181513244218. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"research:gpt-6-luna:5","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.414,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":22.22222222222222,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: HX9sRVWJtZiNvneALnVzs8. Startzeit: 2026-09-22T22:14:05.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.414 und stderr=0.015583544104177553. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: HX9sRVWJtZiNvneALnVzs8. Started at 2026-09-22T22:14:05.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.414 and stderr=0.015583544104177553. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":67.92929292929293,"measurements":[{"observationId":"research:gpt-6-luna:2","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.5609756097560976,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":72.22222222222223,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: 7n3HBUK684GmdT6Tkycu6m. Startzeit: 2026-09-22T22:14:05.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.5609756097560976 und stderr=0.07846686801046543. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: 7n3HBUK684GmdT6Tkycu6m. Started at 2026-09-22T22:14:05.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.5609756097560976 and stderr=0.07846686801046543. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gpt-6-luna:1","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.7894736842105263,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":63.63636363636363,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: iBQxysYR7zPxB6hxvJyTLc. Startzeit: 2026-09-22T22:14:05.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.7894736842105263 und stderr=0.024191503146153773. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: iBQxysYR7zPxB6hxvJyTLc. Started at 2026-09-22T22:14:05.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.7894736842105263 and stderr=0.024191503146153773. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":33.333333333333336,"measurements":[{"observationId":"research:gpt-6-luna:3","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9046717171717171,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":33.333333333333336,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-luna_max. Epoch-Modellbezeichnung: GPT-6 Luna. Anzeigenamenfeld: GPT-6 Luna (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: 7F3QFMAkwnCV62HcvxP96q. Startzeit: 2026-09-22T22:14:05.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9046717171717171 und stderr=0.018537378631915503. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-luna_max; Epoch model label: GPT-6 Luna; Display label field: GPT-6 Luna (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: 7F3QFMAkwnCV62HcvxP96q. Started at 2026-09-22T22:14:05.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9046717171717171 and stderr=0.018537378631915503. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":38.5078776645042,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 38,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 38.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-6-luna-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1581.7308518473817,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":44.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 17.869 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-luna-max. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-luna-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 17,869 published pairwise votes.. Source model ID: lmarena/gpt-6-luna-max. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-6-luna-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 17.869 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 17,869 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 17.869 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 17,869 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":0,"measurements":[{"observationId":"lmarena:search:overall:gpt-6-luna-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1159.5568693390055,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":0,"participants":14,"methodology":{"de":"Arena-Bewertung aus 10.793 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-luna-max. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-luna-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,793 published pairwise votes.. Source model ID: lmarena/gpt-6-luna-max. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-6-luna-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.793 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 10,793 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.793 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 10,793 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"lmarena:overall:gpt-6-luna-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1442.953685098201,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":11.11111111111111,"participants":37,"methodology":{"de":"Arena-Bewertung aus 43.294 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-luna-max. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-luna-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 43,294 published pairwise votes.. Source model ID: lmarena/gpt-6-luna-max. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-6-luna-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 43.294 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 43,294 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 43.294 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 43,294 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":10,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-6-luna-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1240.9714374240696,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":10,"participants":31,"methodology":{"de":"Arena-Bewertung aus 2.833 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-luna-max. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-luna-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,833 published pairwise votes.. Source model ID: lmarena/gpt-6-luna-max. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: gpt-6-luna-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.833 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 2,833 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.833 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 2,833 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":69.64285714285714,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":79.6531791907514,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.64285714285714,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 79,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 79.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":62.49788115641775,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":12.6262626262626,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 12,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 12.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":73.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.28571428571429,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 425.094. Standardfehler: 1.716. Kosten je Aufgabe in USD: 0.029222. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 425.094. Standard error: 1.716. Cost per task in USD: 0.029222. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":13.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.83783783783784,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1965.568. Standardfehler: 1.515. Kosten je Aufgabe in USD: 0.354726. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1965.568. Standard error: 1.515. Cost per task in USD: 0.354726. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":35.44880785413745,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":8.57142857142857,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.47826086956522,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 8,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/6fb13851-40ce-4bda-ad3f-6b38e0c5daa7/gpt-6-luna/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Model snapshot: GPT-6 Luna (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 8.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Luna (Max) (Release gpt-6-luna, Zeile gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Luna (Max) (release gpt-6-luna, row gpt-6-luna, UUID 6fb13851-40ce-4bda-ad3f-6b38e0c5daa7, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":4.286,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.419354838709676,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4559.47. Standardfehler: 2.438. Kosten je Aufgabe in USD: 0.240966. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4559.47. Standard error: 2.438. Cost per task in USD: 0.240966. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":13.157894736842104,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":61.481,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.157894736842104,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 523.805. Standardfehler: 2.593. Kosten je Aufgabe in USD: 0.053236. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 523.805. Standard error: 2.593. Cost per task in USD: 0.053236. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":64.51612903225806,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":42.554,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.51612903225806,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3023.191. Standardfehler: 4.415. Kosten je Aufgabe in USD: 0.600876. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3023.191. Standard error: 4.415. Cost per task in USD: 0.600876. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":76.25,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.9090909090909,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 949.797. Standardfehler: 5.288. Kosten je Aufgabe in USD: 0.133802. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 949.797. Standard error: 5.288. Cost per task in USD: 0.133802. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":77.96610169491525,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":68.515,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.96610169491525,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1041.052. Standardfehler: 2.809. Kosten je Aufgabe in USD: 0.141708. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1041.052. Standard error: 2.809. Cost per task in USD: 0.141708. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":48.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":49.873,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 897.881. Standardfehler: 0.232. Kosten je Aufgabe in USD: 0.124338. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 897.881. Standard error: 0.232. Cost per task in USD: 0.124338. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":48.484848484848484,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":55.556,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.484848484848484,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2147.141. Standardfehler: 8.866. Kosten je Aufgabe in USD: 0.199412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2147.141. Standard error: 8.866. Cost per task in USD: 0.199412. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":50,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.917,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1051.83. Standardfehler: 1.362. Kosten je Aufgabe in USD: 0.303552. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1051.83. Standard error: 1.362. Cost per task in USD: 0.303552. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":45,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":30.288,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2547.747. Standardfehler: 3.194. Kosten je Aufgabe in USD: 0.440296. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2547.747. Standard error: 3.194. Cost per task in USD: 0.440296. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":55.223880597014926,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":44.685,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.223880597014926,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 108.772. Standardfehler: 2.303. Kosten je Aufgabe in USD: 0.007323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 108.772. Standard error: 2.303. Cost per task in USD: 0.007323. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":55.88235294117647,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.71,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.88235294117647,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 169.606. Standardfehler: 1.949. Kosten je Aufgabe in USD: 0.009562. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 169.606. Standard error: 1.949. Cost per task in USD: 0.009562. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":10,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":19.369,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 361.208. Standardfehler: 2.658. Kosten je Aufgabe in USD: 0.025854. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 361.208. Standard error: 2.658. Cost per task in USD: 0.025854. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1734.863. Standardfehler: 0.5. Kosten je Aufgabe in USD: 0.180944. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1734.863. Standard error: 0.5. Cost per task in USD: 0.180944. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":64,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 451.481. Standardfehler: 4.824. Kosten je Aufgabe in USD: 0.041637. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 451.481. Standard error: 4.824. Cost per task in USD: 0.041637. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":60.3448275862069,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":48.091,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.3448275862069,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 91.072. Standardfehler: 3.378. Kosten je Aufgabe in USD: 0.006519. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 91.072. Standard error: 3.378. Cost per task in USD: 0.006519. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":23.68421052631579,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":57.645,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.68421052631579,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3178.908. Standardfehler: 1.285. Kosten je Aufgabe in USD: 0.224168. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3178.908. Standard error: 1.285. Cost per task in USD: 0.224168. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":2.672,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4871.097. Standardfehler: 0.998. Kosten je Aufgabe in USD: 1.550056. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4871.097. Standard error: 0.998. Cost per task in USD: 1.550056. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"vals-swebench-multimodal:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":37.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":14,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 315.874. Standardfehler: 2.212. Kosten je Aufgabe in USD: 0.03193. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 315.874. Standard error: 2.212. Cost per task in USD: 0.03193. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":38.18181818181818,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":19.947,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.18181818181818,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1995.736. Standardfehler: 2.866. Kosten je Aufgabe in USD: 0.190956. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1995.736. Standard error: 2.866. Cost per task in USD: 0.190956. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-6-luna","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":81.649,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.77777777777777,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-luna. Name des Quellmodells: GPT-6 Luna. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2219.515. Standardfehler: 3.377. Kosten je Aufgabe in USD: 1.345703. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-luna. Source model name: GPT-6 Luna. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2219.515. Standard error: 3.377. Cost per task in USD: 1.345703. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":46.054911824179726,"coveredFamilies":40,"totalFamilies":71,"sourceCount":4,"coverage":0.6093253968253969,"lowerBound":28.06242742302697,"upperBound":67.12988774048728,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":51.016977118729265,"coveredFamilies":9,"totalFamilies":15,"sourceCount":3,"coverage":0.5888888888888891,"lowerBound":30.043330969918355,"upperBound":71.15444208102944,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":62.23867595818816,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":35.564957690393236,"upperBound":78.4221005475361,"provisional":true,"configurationsDiffer":true},"math":{"score":63.610463610463604,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":38.16627816627817,"upperBound":78.16627816627816,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":37.330623306233065,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":24.887082204155377,"upperBound":58.22041553748871,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":22.28016688061617,"upperBound":88.94683354728284,"provisional":true,"configurationsDiffer":true},"general":{"score":16.900584795321638,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":13.52046783625731,"upperBound":33.52046783625731,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":23.497321887152392,"upperBound":80.64017903000953,"provisional":true,"configurationsDiffer":true},"legal":{"score":47.49999999999999,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":23.749999999999996,"upperBound":73.75,"provisional":true,"configurationsDiffer":true},"health":{"score":55.5531167690957,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":37.03541117939713,"upperBound":70.36874451273047,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":90.9090909090909,"upperBound":90.9090909090909,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":11.205357142857142,"upperBound":73.70535714285714,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":0,"upperBound":0,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":11.842105263157896,"upperBound":61.8421052631579,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":30.172413793103445,"upperBound":80.17241379310344,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-6-sol","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"research:gpt-6-sol:8","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":1,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: VZAUcNkzeMSMm9paxBhj6B. Startzeit: 2026-09-22T19:59:24.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=1.0 und stderr=0.0. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: VZAUcNkzeMSMm9paxBhj6B. Started at 2026-09-22T19:59:24.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=1.0 and stderr=0.0. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1480.92,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":89.02439024390245,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":61.63369295959864,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 61,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 61.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":25.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 25,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 25.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1508.23,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":85.71428571428571,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":49.4350282485876,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.71428571428571,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 49,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 49.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":87.8048780487805,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":27.116666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":87.8048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":71.42857142857143,"measurements":[{"observationId":"research:gpt-6-sol:6","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.56,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":71.42857142857143,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: N9uFn7EZFsvHNcHEwEy3J6. Startzeit: 2026-10-13T13:41:42.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.56 und stderr=0.049888765156985884. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: N9uFn7EZFsvHNcHEwEy3J6. Started at 2026-10-13T13:41:42.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.56 and stderr=0.049888765156985884. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":55.55555555555556,"measurements":[{"observationId":"research:gpt-6-sol:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.607,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":55.55555555555556,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: donMeEKBZZdqpQsojKWJEm. Startzeit: 2026-09-22T19:59:24.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.607 und stderr=0.015452824654081525. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: donMeEKBZZdqpQsojKWJEm. Started at 2026-09-22T19:59:24.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.607 and stderr=0.015452824654081525. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":89.8989898989899,"measurements":[{"observationId":"research:gpt-6-sol:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.9,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":88.88888888888889,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: jwS3LYLCPbZpHDzzkX6LyV. Startzeit: 2026-09-22T19:59:24.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.9 und stderr=0.051739520574625435. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: jwS3LYLCPbZpHDzzkX6LyV. Started at 2026-09-22T19:59:24.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.9 and stderr=0.051739520574625435. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:gpt-6-sol:3","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.8982456140350877,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: RHjHEPHEsWZqFuMjBTeC6r. Startzeit: 2026-09-22T19:59:24.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.8982456140350877 und stderr=0.01793969072128637. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: RHjHEPHEsWZqFuMjBTeC6r. Started at 2026-09-22T19:59:24.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.8982456140350877 and stderr=0.01793969072128637. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":76.19047619047619,"measurements":[{"observationId":"research:gpt-6-sol:5","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9431818181818182,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":76.19047619047619,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6-sol_max. Epoch-Modellbezeichnung: GPT-6 Sol. Anzeigenamenfeld: GPT-6 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: aJD7ZfFftu2cZoToJhhzF7. Startzeit: 2026-09-22T19:59:24.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9431818181818182 und stderr=0.014938512008159717. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6-sol_max; Epoch model label: GPT-6 Sol; Display label field: GPT-6 Sol (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: aJD7ZfFftu2cZoToJhhzF7. Started at 2026-09-22T19:59:24.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9431818181818182 and stderr=0.014938512008159717. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":89.02439024390245,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":47.914735866543104,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 47,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 47.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-6-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1685.8643231911728,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":83.33333333333333,"participants":37,"methodology":{"de":"Arena-Bewertung aus 4.032 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-sol-max. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 4,032 published pairwise votes.. Source model ID: lmarena/gpt-6-sol-max. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 4.032 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 4,032 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 4.032 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 4,032 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"lmarena:search:overall:gpt-6-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1174.911740373027,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":15.384615384615385,"participants":14,"methodology":{"de":"Arena-Bewertung aus 11.313 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-sol-max. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,313 published pairwise votes.. Source model ID: lmarena/gpt-6-sol-max. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.313 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 11,313 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.313 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 11,313 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":27.77777777777778,"measurements":[{"observationId":"lmarena:overall:gpt-6-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1458.2709182424658,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":27.77777777777778,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.903 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-sol-max. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,903 published pairwise votes.. Source model ID: lmarena/gpt-6-sol-max. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.903 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 11,903 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.903 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 11,903 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":50,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-6-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1270.4040607998948,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":50,"participants":31,"methodology":{"de":"Arena-Bewertung aus 2.657 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6-sol-max. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,657 published pairwise votes.. Source model ID: lmarena/gpt-6-sol-max. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.657 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 2,657 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.657 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 2,657 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":85.71428571428571,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":82.94797687861269,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.71428571428571,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 82,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 82.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":90.05285337602412,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":43.9393939393939,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 43,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 43.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":83.146,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.17857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 298.876. Standardfehler: 1.297. Kosten je Aufgabe in USD: 0.373024. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 298.876. Standard error: 1.297. Cost per task in USD: 0.373024. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":44.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.37837837837837,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2096.128. Standardfehler: 3.535. Kosten je Aufgabe in USD: 5.79322. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2096.128. Standard error: 3.535. Cost per task in USD: 5.79322. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":83.8008415147265,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:c50ea08c-88c0-4eb7-85b8-27f2bbf6d527","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":30,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.95652173913044,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 30,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/c50ea08c-88c0-4eb7-85b8-27f2bbf6d527/gpt-6-sol/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 30.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6 Sol (Max) (Release gpt-6-sol, Zeile gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6 Sol (Max) (release gpt-6-sol, row gpt-6-sol, UUID c50ea08c-88c0-4eb7-85b8-27f2bbf6d527, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":30,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.64516129032258,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4930.428. Standardfehler: 5.517. Kosten je Aufgabe in USD: 5.821477. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4930.428. Standard error: 5.517. Cost per task in USD: 5.821477. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":73.6842105263158,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":74.815,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.6842105263158,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 324.465. Standardfehler: 2.429. Kosten je Aufgabe in USD: 0.662358. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 324.465. Standard error: 2.429. Cost per task in USD: 0.662358. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":88.70967741935483,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":57.195,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.70967741935483,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5136.155. Standardfehler: 4.213. Kosten je Aufgabe in USD: 15.676945. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 5136.155. Standard error: 4.213. Cost per task in USD: 15.676945. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":96.96969696969697,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":77.976,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.96969696969697,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 734.454. Standardfehler: 5.11. Kosten je Aufgabe in USD: 1.338572. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 734.454. Standard error: 5.11. Cost per task in USD: 1.338572. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":84.7457627118644,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":71.53,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.7457627118644,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 621.736. Standardfehler: 2.351. Kosten je Aufgabe in USD: 1.276882. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 621.736. Standard error: 2.351. Cost per task in USD: 1.276882. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":45,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":49.05,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 539.907. Standardfehler: 0.58. Kosten je Aufgabe in USD: 2.122121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 539.907. Standard error: 0.58. Cost per task in USD: 2.122121. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":75.75757575757575,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":82.611,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.75757575757575,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1837.553. Standardfehler: 9.24. Kosten je Aufgabe in USD: 2.703348. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1837.553. Standard error: 9.24. Cost per task in USD: 2.703348. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":37.704918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.704918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 820.097. Standardfehler: 0.823. Kosten je Aufgabe in USD: 3.358015. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 820.097. Standard error: 0.823. Cost per task in USD: 3.358015. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":41.666666666666664,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":28.846,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1449.451. Standardfehler: 3.149. Kosten je Aufgabe in USD: 4.897717. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1449.451. Standard error: 3.149. Cost per task in USD: 4.897717. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":61.19402985074627,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":47.072,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.19402985074627,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 62.714. Standardfehler: 2.119. Kosten je Aufgabe in USD: 0.085827. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 62.714. Standard error: 2.119. Cost per task in USD: 0.085827. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":47.05882352941177,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":82.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.05882352941177,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 81.135. Standardfehler: 1.942. Kosten je Aufgabe in USD: 0.083138. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 81.135. Standard error: 1.942. Cost per task in USD: 0.083138. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":45,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":30.18,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 373.426. Standardfehler: 3.088. Kosten je Aufgabe in USD: 0.460931. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 373.426. Standard error: 3.088. Cost per task in USD: 0.460931. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":83.6734693877551,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":2,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.6734693877551,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2771.488. Standardfehler: 0.992. Kosten je Aufgabe in USD: 11.672651. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2771.488. Standard error: 0.992. Cost per task in USD: 11.672651. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":70.51282051282051,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":83,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.51282051282051,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 304.509. Standardfehler: 3.775. Kosten je Aufgabe in USD: 0.476321. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 304.509. Standard error: 3.775. Cost per task in USD: 0.476321. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":41.37931034482759,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":44.793,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.37931034482759,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 31.419. Standardfehler: 3.364. Kosten je Aufgabe in USD: 0.061332. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 30000. Latency in seconds: 31.419. Standard error: 3.364. Cost per task in USD: 0.061332. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 30000","en":"Reasoning effort: max. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":19.736842105263158,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":56.631,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.736842105263158,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1862.494. Standardfehler: 1.289. Kosten je Aufgabe in USD: 2.491613. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1862.494. Standard error: 1.289. Cost per task in USD: 2.491613. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":85.41666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":30.534,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.41666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3807.782. Standardfehler: 2.851. Kosten je Aufgabe in USD: 20.651048. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3807.782. Standard error: 2.851. Cost per task in USD: 20.651048. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":29.09090909090909,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":15.002,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.09090909090909,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1052.036. Standardfehler: 2.575. Kosten je Aufgabe in USD: 2.277613. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1052.036. Standard error: 2.575. Cost per task in USD: 2.277613. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-6-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":87.824,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.88888888888889,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6-sol. Name des Quellmodells: GPT-6 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2336.393. Standardfehler: 2.526. Kosten je Aufgabe in USD: 26.356012. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6-sol. Source model name: GPT-6 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2336.393. Standard error: 2.526. Cost per task in USD: 26.356012. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":60.09428008792914,"coveredFamilies":38,"totalFamilies":71,"sourceCount":4,"coverage":0.5998015873015873,"lowerBound":36.04464458448607,"upperBound":76.06448585432734,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.4555555555555557,"lowerBound":38.61385538544365,"upperBound":93.05829982988809,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":85.15534262485482,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":48.66019578563133,"upperBound":91.51733864277419,"provisional":true,"configurationsDiffer":true},"math":{"score":83.77363377363378,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":50.26418026418027,"upperBound":90.26418026418025,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":71.68021680216803,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":47.78681120144535,"upperBound":81.12014453477869,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":28.179442508710796,"upperBound":94.84610917537746,"provisional":true,"configurationsDiffer":true},"general":{"score":57.96966374269005,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":46.37573099415204,"upperBound":66.37573099415204,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":22.69095311468193,"upperBound":79.83381025753907,"provisional":true,"configurationsDiffer":true},"legal":{"score":39.685792349726775,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":19.842896174863387,"upperBound":69.84289617486338,"provisional":true,"configurationsDiffer":true},"health":{"score":54.12642669007902,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":36.08428446005268,"upperBound":69.41761779338601,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":96.96969696969697,"upperBound":96.96969696969697,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":23.21428571428571,"upperBound":85.71428571428571,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":15.384615384615383,"upperBound":15.384615384615383,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":9.868421052631579,"upperBound":59.868421052631575,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":20.689655172413794,"upperBound":70.6896551724138,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-6.1-sol","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":90.9090909090909,"measurements":[{"observationId":"research:gpt-6.1-sol:5","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":1,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":90.9090909090909,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6.1-sol_max. Epoch-Modellbezeichnung: GPT-6.1 Sol. Anzeigenamenfeld: GPT-6.1 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: Di7y6sES4PGQbXNkXYhdqM. Startzeit: 2026-09-29T22:00:34.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=1.0 und stderr=0.0. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6.1-sol_max; Epoch model label: GPT-6.1 Sol; Display label field: GPT-6.1 Sol (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: Di7y6sES4PGQbXNkXYhdqM. Started at 2026-09-29T22:00:34.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=1.0 and stderr=0.0. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":77.63157894736842,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":50,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":77.63157894736842,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 50,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 50.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1557,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":88.41463414634147,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":83,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.41463414634147,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":64.86844882398171,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 64,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 64.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":49.1209333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 49,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 49.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":98.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":31,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 31,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 31.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":1593.85,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":97.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":41.53333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":95.23809523809524,"measurements":[{"observationId":"research:gpt-6.1-sol:3","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.68,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":95.23809523809524,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6.1-sol_max. Epoch-Modellbezeichnung: GPT-6.1 Sol. Anzeigenamenfeld: GPT-6.1 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: KkGEgrjYDnofMoUJBCykcH. Startzeit: 2026-10-13T12:40:12.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.68 und stderr=0.04688261722621507. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6.1-sol_max; Epoch model label: GPT-6.1 Sol; Display label field: GPT-6.1 Sol (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: KkGEgrjYDnofMoUJBCykcH. Started at 2026-10-13T12:40:12.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.68 and stderr=0.04688261722621507. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":88.88888888888889,"measurements":[{"observationId":"research:gpt-6.1-sol:4","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.739,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":88.88888888888889,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6.1-sol_max. Epoch-Modellbezeichnung: GPT-6.1 Sol. Anzeigenamenfeld: GPT-6.1 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: dfT5VD7cQug8XW6GQHyJUF. Startzeit: 2026-09-29T22:00:34.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.739 und stderr=0.013895037677965185. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6.1-sol_max; Epoch model label: GPT-6.1 Sol; Display label field: GPT-6.1 Sol (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: dfT5VD7cQug8XW6GQHyJUF. Started at 2026-09-29T22:00:34.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.739 and stderr=0.013895037677965185. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":0.5,"score":97.72727272727273,"measurements":[{"observationId":"research:gpt-6.1-sol:0","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.9368421052631579,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":97.72727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6.1-sol_max. Epoch-Modellbezeichnung: GPT-6.1 Sol. Anzeigenamenfeld: GPT-6.1 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: aBoQLaqnXsLPrsbN8wYgUp. Startzeit: 2026-09-29T22:00:34.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.9368421052631579 und stderr=0.014434038295343787. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6.1-sol_max; Epoch model label: GPT-6.1 Sol; Display label field: GPT-6.1 Sol (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: aBoQLaqnXsLPrsbN8wYgUp. Started at 2026-09-29T22:00:34.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.9368421052631579 and stderr=0.014434038295343787. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":88.0952380952381,"measurements":[{"observationId":"research:gpt-6.1-sol:2","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9539141414141414,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":88.0952380952381,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: gpt-6.1-sol_max. Epoch-Modellbezeichnung: GPT-6.1 Sol. Anzeigenamenfeld: GPT-6.1 Sol (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: LspviLnbeZ82WCngWp6rwR. Startzeit: 2026-09-29T22:00:34.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9539141414141414 und stderr=0.013780748450364692. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: gpt-6.1-sol_max; Epoch model label: GPT-6.1 Sol; Display label field: GPT-6.1 Sol (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: LspviLnbeZ82WCngWp6rwR. Started at 2026-09-29T22:00:34.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9539141414141414 and stderr=0.013780748450364692. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":52.919369786839695,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 52,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 52.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"lmarena:webdev:overall:gpt-6.1-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1760.0883372906792,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":91.66666666666667,"participants":37,"methodology":{"de":"Arena-Bewertung aus 2.592 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6.1-sol-max. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6.1-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,592 published pairwise votes.. Source model ID: lmarena/gpt-6.1-sol-max. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6.1-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.592 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 2,592 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.592 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 2,592 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"lmarena:search:overall:gpt-6.1-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1233.1023499390756,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":69.23076923076923,"participants":14,"methodology":{"de":"Arena-Bewertung aus 5.555 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6.1-sol-max. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6.1-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 5,555 published pairwise votes.. Source model ID: lmarena/gpt-6.1-sol-max. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6.1-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 5.555 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 5,555 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 5.555 veröffentlichten Paarvergleichen.. Harness: Arena Search. Reasoning-Stufe: max","en":"Arena rating from 5,555 published pairwise votes.. Harness: Arena Search. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":69.44444444444444,"measurements":[{"observationId":"lmarena:overall:gpt-6.1-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1483.9589874960711,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":69.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 5.762 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6.1-sol-max. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6.1-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 5,762 published pairwise votes.. Source model ID: lmarena/gpt-6.1-sol-max. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6.1-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 5.762 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 5,762 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 5.762 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 5,762 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":93.33333333333333,"measurements":[{"observationId":"lmarena:vision_style_control:overall:gpt-6.1-sol-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1294.0517712311234,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":93.33333333333333,"participants":31,"methodology":{"de":"Arena-Bewertung aus 1.813 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/gpt-6.1-sol-max. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: gpt-6.1-sol-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 1,813 published pairwise votes.. Source model ID: lmarena/gpt-6.1-sol-max. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: gpt-6.1-sol-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 1.813 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 1,813 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 1.813 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 1,813 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":98.21428571428571,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":85.9537572254335,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.21428571428571,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 86,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 86.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":94.79235332893869,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":56.0606060606061,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 56,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 56.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":55.051,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.48648648648648,"participants":38,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2711.667. Standardfehler: 1.821. Kosten je Aufgabe in USD: 1.72496. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2711.667. Standard error: 1.821. Cost per task in USD: 1.72496. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":96.21318373071529,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Max","retrievedAt":"2026-10-19","value":58.0952380952381,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.65217391304348,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Modellstand: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 58,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/b25ac058-1e33-4c31-bbc9-a20c3ad0717a/gpt-6-1-sol/gpt-6-1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Model snapshot: GPT-6.1 Sol (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 58.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: GPT-6.1 Sol (Max) (Release gpt-6-1-sol, Zeile gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: GPT-6.1 Sol (Max) (release gpt-6-1-sol, row gpt-6-1-sol, UUID b25ac058-1e33-4c31-bbc9-a20c3ad0717a, reasoning=true, source reasoning-token configuration budget=0, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":52.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.7741935483871,"participants":32,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5003.987. Standardfehler: 6.009. Kosten je Aufgabe in USD: 3.436948. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 5003.987. Standard error: 6.009. Cost per task in USD: 3.436948. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":94.73684210526316,"measurements":[{"observationId":"vals-biomysterybench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":79.63,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.73684210526316,"participants":20,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 530.7. Standardfehler: 0.741. Kosten je Aufgabe in USD: 0.372147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 530.7. Standard error: 0.741. Cost per task in USD: 0.372147. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: Terminus 2. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":95.16129032258064,"measurements":[{"observationId":"vals-code-migration:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":65.122,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.16129032258064,"participants":63,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5781.916. Standardfehler: 4.362. Kosten je Aufgabe in USD: 6.505708. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 5781.916. Standard error: 4.362. Cost per task in USD: 6.505708. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":6.0606060606060606,"measurements":[{"observationId":"vals-cyberbench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":39.286,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.0606060606060606,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 221.992. Standardfehler: 2.742. Kosten je Aufgabe in USD: 0.285372. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 221.992. Standard error: 2.742. Cost per task in USD: 0.285372. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":81.35593220338983,"measurements":[{"observationId":"vals-emb:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":70.773,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.35593220338983,"participants":60,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1107.811. Standardfehler: 2.712. Kosten je Aufgabe in USD: 1.143199. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1107.811. Standard error: 2.712. Cost per task in USD: 1.143199. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":56.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":52.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 925.46. Standardfehler: 0.308. Kosten je Aufgabe in USD: 1.619399. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 925.46. Standard error: 0.308. Cost per task in USD: 1.619399. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":96.96969696969697,"measurements":[{"observationId":"vals-ioi:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":96.889,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.96969696969697,"participants":34,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1890.714. Standardfehler: 3.111. Kosten je Aufgabe in USD: 1.261598. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1890.714. Standard error: 3.111. Cost per task in USD: 1.261598. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":64.75409836065573,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":5.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.75409836065573,"participants":62,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2594.18. Standardfehler: 1.755. Kosten je Aufgabe in USD: 3.933528. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2594.18. Standard error: 1.755. Cost per task in USD: 3.933528. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":60,"measurements":[{"observationId":"vals-legal-research:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":38.462,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60,"participants":61,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2060.48. Standardfehler: 3.381. Kosten je Aufgabe in USD: 3.384561. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 2060.48. Standard error: 3.381. Cost per task in USD: 3.384561. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":71.64179104477611,"measurements":[{"observationId":"vals-medcode:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":48.839,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.64179104477611,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 115.342. Standardfehler: 2.12. Kosten je Aufgabe in USD: 0.084726. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 115.342. Standard error: 2.12. Cost per task in USD: 0.084726. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":75,"measurements":[{"observationId":"vals-medscribe:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":86.453,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 169.619. Standardfehler: 1.898. Kosten je Aufgabe in USD: 0.100902. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 169.619. Standard error: 1.898. Cost per task in USD: 0.100902. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":80,"measurements":[{"observationId":"vals-mysterymechanism:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":46.396,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80,"participants":21,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 437.025. Standardfehler: 3.355. Kosten je Aufgabe in USD: 0.284881. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 437.025. Standard error: 3.355. Cost per task in USD: 0.284881. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":90.81632653061224,"measurements":[{"observationId":"vals-programbench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":3,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.81632653061224,"participants":50,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4223.187. Standardfehler: 1.209. Kosten je Aufgabe in USD: 2.868128. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4223.187. Standard error: 1.209. Cost per task in USD: 2.868128. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":89.74358974358974,"measurements":[{"observationId":"vals-proof-bench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":99,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.74358974358974,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 284.697. Standardfehler: 1. Kosten je Aufgabe in USD: 0.233057. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 284.697. Standard error: 1. Cost per task in USD: 0.233057. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":51.724137931034484,"measurements":[{"observationId":"vals-sage:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":46.538,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.724137931034484,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 50.219. Standardfehler: 3.462. Kosten je Aufgabe in USD: 0.061245. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 50.219. Standard error: 3.462. Cost per task in USD: 0.061245. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":28.94736842105263,"measurements":[{"observationId":"vals-snap:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":59.337,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.94736842105263,"participants":39,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3024.455. Standardfehler: 1.278. Kosten je Aufgabe in USD: 0.690795. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 3024.455. Standard error: 1.278. Cost per task in USD: 0.690795. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":95.83333333333333,"measurements":[{"observationId":"vals-sre-bench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":50.763,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.83333333333333,"participants":25,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1952.046. Standardfehler: 3.095. Kosten je Aufgabe in USD: 2.685175. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1952.046. Standard error: 3.095. Cost per task in USD: 2.685175. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"vals-swebench-multimodal:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":43.542,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":14,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 415.53. Standardfehler: 2.265. Kosten je Aufgabe in USD: 0.35065. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 415.53. Standard error: 2.265. Cost per task in USD: 0.35065. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-tax-agent-bench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":21.754,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":56,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1213.878. Standardfehler: 2.977. Kosten je Aufgabe in USD: 1.323601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 1213.878. Standard error: 2.977. Cost per task in USD: 1.323601. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":91.66666666666667,"measurements":[{"observationId":"vals-vibe-code-bench:overall:openai/gpt-6.1-sol","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":88.931,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.66666666666667,"participants":73,"methodology":{"de":"Kennung des Quellmodells: openai/gpt-6.1-sol. Name des Quellmodells: GPT-6.1 Sol. Anbieter laut Quelle: OpenAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4489.755. Standardfehler: 1.893. Kosten je Aufgabe in USD: 6.234794. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/gpt-6.1-sol. Source model name: GPT-6.1 Sol. Provider reported by source: OpenAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000. Latency in seconds: 4489.755. Standard error: 1.893. Cost per task in USD: 6.234794. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Reasoning effort: max. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":67.34421896463805,"coveredFamilies":40,"totalFamilies":71,"sourceCount":4,"coverage":0.6013888888888889,"lowerBound":40.50006501623371,"upperBound":80.36117612734483,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":89.05875914539354,"coveredFamilies":9,"totalFamilies":15,"sourceCount":3,"coverage":0.5777777777777778,"lowerBound":51.45617195067182,"upperBound":93.67839417289403,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":93.35801393728224,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":53.34743653558985,"upperBound":96.20457939273271,"provisional":true,"configurationsDiffer":true},"math":{"score":91.8065268065268,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.5,"lowerBound":45.9032634032634,"upperBound":95.9032634032634,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":93.22493224932249,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":62.149954832881654,"upperBound":95.48328816621499,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":28.623769790329483,"upperBound":95.29043645699616,"provisional":true,"configurationsDiffer":true},"general":{"score":85.00365497076024,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":68.00292397660819,"upperBound":88.00292397660819,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":26.211020617800276,"upperBound":83.35387776065741,"provisional":true,"configurationsDiffer":true},"legal":{"score":62.37704918032786,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":31.18852459016393,"upperBound":81.18852459016392,"provisional":true,"configurationsDiffer":true},"health":{"score":73.32089552238806,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":48.880597014925364,"upperBound":82.2139303482587,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":6.0606060606060606,"upperBound":6.0606060606060606,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":35.61011904761905,"upperBound":98.11011904761905,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":69.23076923076923,"upperBound":69.23076923076923,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":14.473684210526315,"upperBound":64.47368421052632,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":25.862068965517242,"upperBound":75.86206896551724,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-oss-120b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":52,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 52,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 52.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":3.09734513274336,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.6923076923076925,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 3,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 3.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":2.4390243902439024,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0.19906990691605594,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.4390243902439024,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 4,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 4.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":608.65,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":65.21739130434783,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":68.9795918367347,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.21739130434783,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 69,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 69.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":5.649717514124291,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 5,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 5.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":-49.25,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":31.57894736842105,"measurements":[{"observationId":"vals-gpqa:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":78.536,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.57894736842105,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 56.176. Standardfehler: 2.253. Kosten je Aufgabe in USD: 0.006212. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 56.176. Standard error: 2.253. Cost per task in USD: 0.006212. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":19.6014828544949,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 19,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 19.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":1.694915254237288,"measurements":[{"observationId":"vals-swebench:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":33.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.694915254237288,"participants":60,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 256.464. Standardfehler: 2.114. Kosten je Aufgabe in USD: 0.171583. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Maximum output tokens: 32768. Latency in seconds: 256.464. Standard error: 2.114. Cost per task in USD: 0.171583. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 32768","en":"Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 32768","en":"Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":16.238385598141697,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:f0083258-8646-45b8-8082-7aaf6c2ea82a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-120b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f0083258-8646-45b8-8082-7aaf6c2ea82a/gpt-oss-120b/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-120b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-120b (High) (Release gpt-oss-120b, Zeile gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-120b (High) (release gpt-oss-120b, row gpt-oss-120b, UUID f0083258-8646-45b8-8082-7aaf6c2ea82a, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":19.101,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.904761904761905,"participants":43,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 490.948. Standardfehler: 4.19. Kosten je Aufgabe in USD: 0.131646. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Maximum output tokens: 32768. Latency in seconds: 490.948. Standard error: 4.19. Cost per task in USD: 0.131646. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 32768","en":"Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 32768","en":"Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":24.324324324324323,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":58.236,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.324324324324323,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 42.722. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.009601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 42.722. Standard error: 0.968. Cost per task in USD: 0.009601. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":55.84415584415584,"measurements":[{"observationId":"vals-lcb:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":83.234,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.84415584415584,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 81.703. Standardfehler: 1.055. Kosten je Aufgabe in USD: 0.008486. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 81.703. Standard error: 1.055. Cost per task in USD: 0.008486. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":75.938,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.666666666666668,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7.852. Standardfehler: 0.517. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7.852. Standard error: 0.517. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":93.18181818181819,"measurements":[{"observationId":"vals-math500:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.18181818181819,"participants":23,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16.275. Standardfehler: 0.993. Kosten je Aufgabe in USD: 0.001381. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 16.275. Standard error: 0.993. Cost per task in USD: 0.001381. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":51.282051282051285,"measurements":[{"observationId":"vals-medqa:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":91.36,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.282051282051285,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12.383. Standardfehler: 0.24. Kosten je Aufgabe in USD: 0.000607. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 12.383. Standard error: 0.24. Cost per task in USD: 0.000607. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":75.86206896551724,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":92.036,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.86206896551724,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5.793. Standardfehler: 1.642. Kosten je Aufgabe in USD: 0.000659. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5.793. Standard error: 1.642. Cost per task in USD: 0.000659. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":79.166,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 35.41. Standardfehler: 0.395. Kosten je Aufgabe in USD: 0.001438. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 35.41. Standard error: 0.395. Cost per task in USD: 0.001438. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":25,"measurements":[{"observationId":"vals-poker-agent:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1015.3311053208165,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":9,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 13.452568330134357. Standardfehler: 0. Kosten je Aufgabe in USD: 0.004534421914446. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 13.452568330134357. Standard error: 0. Cost per task in USD: 0.004534421914446. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":35.526315789473685,"measurements":[{"observationId":"vals-tax-eval-v2:overall:fireworks/gpt-oss-120b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.586,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-120b. Name des Quellmodells: GPT OSS 120B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 47.661. Standardfehler: 0.878. Kosten je Aufgabe in USD: 0.002958. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-120b. Source model name: GPT OSS 120B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 47.661. Standard error: 0.878. Cost per task in USD: 0.002958. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":22,"totalFamilies":71,"sourceCount":2,"coverage":0.22919501133786854,"lowerBound":7.638092522127805,"upperBound":84.71859138834095,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":2,"coverage":0.14444444444444451,"lowerBound":4.501146924669043,"upperBound":90.0567024802246,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":10.53090042178617,"upperBound":67.67375756464331,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":33.80877742946709,"upperBound":93.80877742946709,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":15.18847006651885,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":10.125646711012566,"upperBound":43.4589800443459,"provisional":true,"configurationsDiffer":true},"agentic":{"score":10.225140712945592,"coveredFamilies":6,"totalFamilies":12,"sourceCount":2,"coverage":0.5,"lowerBound":5.112570356472796,"upperBound":55.112570356472794,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":13.043478260869566,"upperBound":93.04347826086956,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":8.550091444828288,"upperBound":79.97866287339971,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":4.166666666666667,"upperBound":79.16666666666667,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":17.094017094017097,"upperBound":83.76068376068378,"provisional":true,"configurationsDiffer":true}}},{"modelId":"gpt-oss-20b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":34.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 34,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 34.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0.737463126843658,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":0.6097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0.19111706206154658,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0.6097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":16.463414634146343,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":16.463414634146343,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 2,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 2.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":339.96,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":52.17391304347826,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":65.10204081632651,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.17391304347826,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 65,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 65.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":3.658536585365854,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":-63.05,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.658536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":18.42105263157895,"measurements":[{"observationId":"vals-gpqa:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":68.94,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.42105263157895,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 145.092. Standardfehler: 2.465. Kosten je Aufgabe in USD: 0.006738. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 145.092. Standard error: 2.465. Cost per task in USD: 0.006738. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":10.982391102873,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 11,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 11.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:36f73aaf-d38a-4b56-a2b3-d04d17186910","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: OpenAI. Modellstand: gpt-oss-20b (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/36f73aaf-d38a-4b56-a2b3-d04d17186910/gpt-oss-20b/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: OpenAI. Model snapshot: gpt-oss-20b (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: gpt-oss-20b (High) (Release gpt-oss-20b, Zeile gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: gpt-oss-20b (High) (release gpt-oss-20b, row gpt-oss-20b, UUID 36f73aaf-d38a-4b56-a2b3-d04d17186910, reasoning=true, source reasoning-token configuration budget=2000, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":14.864864864864865,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":53.147,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.864864864864865,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 33.968. Standardfehler: 0.979. Kosten je Aufgabe in USD: 0.004549. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 33.968. Standard error: 0.979. Cost per task in USD: 0.004549. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":33.76623376623377,"measurements":[{"observationId":"vals-lcb:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":80.387,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.76623376623377,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 109.453. Standardfehler: 1.078. Kosten je Aufgabe in USD: 0.006706. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 109.453. Standard error: 1.078. Cost per task in USD: 0.006706. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":10.256410256410257,"measurements":[{"observationId":"vals-legal-bench:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":70.849,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.256410256410257,"participants":79,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7.187. Standardfehler: 0.61. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 7.187. Standard error: 0.61. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"vals-math500:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.27272727272727,"participants":23,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 31.472. Standardfehler: 1.045. Kosten je Aufgabe in USD: 0.000955. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 31.472. Standard error: 1.045. Cost per task in USD: 0.000955. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":28.205128205128204,"measurements":[{"observationId":"vals-medqa:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":82.875,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.205128205128204,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 25.351. Standardfehler: 0.344. Kosten je Aufgabe in USD: 0.000335. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 25.351. Standard error: 0.344. Cost per task in USD: 0.000335. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":37.93103448275862,"measurements":[{"observationId":"vals-mgsm:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":89.018,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.93103448275862,"participants":30,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5.791. Standardfehler: 1.89. Kosten je Aufgabe in USD: 0.000453. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 5.791. Standard error: 1.89. Cost per task in USD: 0.000453. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-mmlu-pro:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":71.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":78,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 30.207. Standardfehler: 0.447. Kosten je Aufgabe in USD: 0.00062. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 30.207. Standard error: 0.447. Cost per task in USD: 0.00062. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":6.578947368421052,"measurements":[{"observationId":"vals-tax-eval-v2:overall:fireworks/gpt-oss-20b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":63.696,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.578947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: fireworks/gpt-oss-20b. Name des Quellmodells: GPT OSS 20B. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 49.004. Standardfehler: 0.936. Kosten je Aufgabe in USD: 0.002254. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/gpt-oss-20b. Source model name: GPT OSS 20B. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 49.004. Standard error: 0.936. Cost per task in USD: 0.002254. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":19,"totalFamilies":71,"sourceCount":2,"coverage":0.21411564625850346,"lowerBound":4.521341967127488,"upperBound":83.10977734127714,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":2.8405131453911956,"upperBound":92.84051314539118,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":6.2809462681092985,"upperBound":63.423803410966435,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":23.040752351097176,"upperBound":83.04075235109718,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":6.3747228381374725,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":4.249815225424982,"upperBound":37.58314855875832,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":1.4227642276422765,"upperBound":68.08943089430895,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":10.434782608695652,"upperBound":90.43478260869566,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":3.0634017476122737,"upperBound":74.4919731761837,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":2.5641025641025643,"upperBound":77.56410256410257,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.401709401709402,"upperBound":76.06837606837607,"provisional":true,"configurationsDiffer":true}}},{"modelId":"granite-4-2-30b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"granite-4-2-3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"granite-4-2-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"granite-4.0-h-micro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"granite-4.1-8b","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":24.63768115942029,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:82ed9bd2-c97b-4c35-9312-94bb72001e36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":38.6394557823129,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.63768115942029,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Granite 4.1 8B (Release granite-4-1-8b, Zeile granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/82ed9bd2-c97b-4c35-9312-94bb72001e36/granite-4-1-8b/granite-4-1-8b. Name des Quellmodells: Granite 4.1 8B. Anbieter laut Quelle: IBM. Modellstand: Granite 4.1 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 38,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Granite 4.1 8B (release granite-4-1-8b, row granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/82ed9bd2-c97b-4c35-9312-94bb72001e36/granite-4-1-8b/granite-4-1-8b. Source model name: Granite 4.1 8B. Provider reported by source: IBM. Model snapshot: Granite 4.1 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 38.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Granite 4.1 8B (Release granite-4-1-8b, Zeile granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Granite 4.1 8B (release granite-4-1-8b, row granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Granite 4.1 8B (Release granite-4-1-8b, Zeile granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Granite 4.1 8B (release granite-4-1-8b, row granite-4-1-8b, UUID 82ed9bd2-c97b-4c35-9312-94bb72001e36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.35196687370600416,"upperBound":98.92339544513459,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.927536231884058,"upperBound":84.92753623188406,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-4-20-non-reasoning","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"grok-4.20-multi-agent","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"grok-4.20-reasoning","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":65.78947368421052,"measurements":[{"observationId":"vals-gpqa:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":88.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.78947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 59.421. Standardfehler: 1.594. Kosten je Aufgabe in USD: 0.034461. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 59.421. Standard error: 1.594. Cost per task in USD: 0.034461. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":48.93617021276596,"measurements":[{"observationId":"vals-mmmu:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.468,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.93617021276596,"participants":48,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 44.461. Standardfehler: 0.893. Kosten je Aufgabe in USD: 0.026669. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 44.461. Standard error: 0.893. Cost per task in USD: 0.026669. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":33.898305084745765,"measurements":[{"observationId":"vals-swebench:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":72.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.898305084745765,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 134.759. Standardfehler: 2.006. Kosten je Aufgabe in USD: 0.515946. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 134.759. Standard error: 2.006. Cost per task in USD: 0.515946. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":30.654761904761905,"measurements":[{"observationId":"vals-terminal-bench-2:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":40.449,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.80952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 161.408. Standardfehler: 5.232. Kosten je Aufgabe in USD: 0.154109. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 161.408. Standard error: 5.232. Cost per task in USD: 0.154109. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":44.195,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.5,"participants":57,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 305.726. Standardfehler: 0.991. Kosten je Aufgabe in USD: 0.407362. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 305.726. Standard error: 0.991. Cost per task in USD: 0.407362. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":1.6129032258064515,"measurements":[{"observationId":"vals-code-migration:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":0.314,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.6129032258064515,"participants":63,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 373.673. Standardfehler: 0.11. Kosten je Aufgabe in USD: 0.289486. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 373.673. Standard error: 0.11. Cost per task in USD: 0.289486. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":62.16216216216216,"measurements":[{"observationId":"vals-corp-fin-v2:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":63.675,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.16216216216216,"participants":75,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 9.67. Standardfehler: 0.947. Kosten je Aufgabe in USD: 0.163386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 9.67. Standard error: 0.947. Cost per task in USD: 0.163386. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":5.084745762711864,"measurements":[{"observationId":"vals-emb:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":11.739,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.084745762711864,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 367.615. Standardfehler: 1.553. Kosten je Aufgabe in USD: 0.469926. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 367.615. Standard error: 1.553. Cost per task in USD: 0.469926. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":5,"measurements":[{"observationId":"vals-finance-agent-v2:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":28.492,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 351.413. Standardfehler: 0.315. Kosten je Aufgabe in USD: 0.791809. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 351.413. Standard error: 0.315. Cost per task in USD: 0.791809. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":64.93506493506493,"measurements":[{"observationId":"vals-lcb:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":84.265,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.93506493506493,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 76.09. Standardfehler: 1.028. Kosten je Aufgabe in USD: 0.05249. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 76.09. Standard error: 1.028. Cost per task in USD: 0.05249. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 502.903. Standardfehler: 0. Kosten je Aufgabe in USD: 0.986814. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 502.903. Standard error: 0. Cost per task in USD: 0.986814. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":20.512820512820515,"measurements":[{"observationId":"vals-legal-bench:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":77.738,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.512820512820515,"participants":79,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 7.574. Standardfehler: 0.48. Kosten je Aufgabe in USD: 0.006065. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 7.574. Standard error: 0.48. Cost per task in USD: 0.006065. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":15.833333333333334,"measurements":[{"observationId":"vals-legal-research:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":13.942,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.833333333333334,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 244.487. Standardfehler: 2.408. Kosten je Aufgabe in USD: 0.261499. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 244.487. Standard error: 2.408. Cost per task in USD: 0.261499. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":8.955223880597014,"measurements":[{"observationId":"vals-medcode:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":32.156,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.955223880597014,"participants":68,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 16.546. Standardfehler: 2.124. Kosten je Aufgabe in USD: 0.036189. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 16.546. Standard error: 2.124. Cost per task in USD: 0.036189. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"vals-medqa:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":94.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":40,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 12.225. Standardfehler: 0.207. Kosten je Aufgabe in USD: 0.009106. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 12.225. Standard error: 0.207. Cost per task in USD: 0.009106. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":4.411764705882353,"measurements":[{"observationId":"vals-medscribe:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":63.412,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.411764705882353,"participants":69,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 18.596. Standardfehler: 2.095. Kosten je Aufgabe in USD: 0.031303. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 18.596. Standard error: 2.095. Cost per task in USD: 0.031303. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":61.03896103896104,"measurements":[{"observationId":"vals-mmlu-pro:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.254,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.03896103896104,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 11.654. Standardfehler: 0.34. Kosten je Aufgabe in USD: 0.012836. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 11.654. Standard error: 0.34. Cost per task in USD: 0.012836. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":1.9230769230769231,"measurements":[{"observationId":"vals-mortgage-tax:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":45.35,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.9230769230769231,"participants":53,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 11.124. Standardfehler: 0.988. Kosten je Aufgabe in USD: 0.024144. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 11.124. Standard error: 0.988. Cost per task in USD: 0.024144. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":22.413793103448278,"measurements":[{"observationId":"vals-sage:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":38.242,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.413793103448278,"participants":59,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 66.358. Standardfehler: 3.425. Kosten je Aufgabe in USD: 0.063231. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 66.358. Standard error: 3.425. Cost per task in USD: 0.063231. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":12.727272727272727,"measurements":[{"observationId":"vals-tax-agent-bench:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":6.952,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.727272727272727,"participants":56,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 89.262. Standardfehler: 1.75. Kosten je Aufgabe in USD: 0.119262. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 89.262. Standard error: 1.75. Cost per task in USD: 0.119262. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":61.8421052631579,"measurements":[{"observationId":"vals-tax-eval-v2:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.121,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.8421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 13.394. Standardfehler: 0.859. Kosten je Aufgabe in USD: 0.017152. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 13.394. Standard error: 0.859. Cost per task in USD: 0.017152. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"vals-vibe-code-bench:overall:grok/grok-4.20-0309-reasoning","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":4.063,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.11111111111111,"participants":73,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.20-0309-reasoning. Name des Quellmodells: Grok 4.20 Reasoning. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000. Latenz in Sekunden: 301.651. Standardfehler: 2.058. Kosten je Aufgabe in USD: 0.771885. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.20-0309-reasoning. Source model name: Grok 4.20 Reasoning. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000. Latency in seconds: 301.651. Standard error: 2.058. Cost per task in USD: 0.771885. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Harness: OpenHands. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 2000000","en":"Harness: OpenHands. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 2000000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":21,"totalFamilies":71,"sourceCount":1,"coverage":0.27803287981859415,"lowerBound":8.032397176920341,"upperBound":80.22910919506093,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":1,"coverage":0.25555555555555565,"lowerBound":6.988432385507291,"upperBound":81.43287682995172,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":4.6992481203007515,"upperBound":97.55639097744361,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":20.346320346320347,"upperBound":87.01298701298703,"provisional":true,"configurationsDiffer":true},"finance":{"score":24.78989380639693,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":21.24848040548308,"upperBound":35.53419469119737,"provisional":true,"configurationsDiffer":true},"legal":{"score":15.667297183690629,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":11.750472887767971,"upperBound":36.75047288776797,"provisional":true,"configurationsDiffer":true},"health":{"score":30.09668850318543,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":30.09668850318543,"upperBound":30.09668850318543,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":6.117021276595745,"upperBound":93.61702127659575,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":11.206896551724139,"upperBound":61.20689655172414,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-4.3","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":3.9473684210526314,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":8.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.9473684210526314,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 8,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 8.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":761.88,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":33.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":73,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 73,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 73.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":17.0353982300885,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":38.46153846153846,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 17,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 17.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1.375964614032744,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 1,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 1.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":5.800000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 5,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 5.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":937.89,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":98.55072463768116,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":81.2925170068027,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.55072463768116,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 81,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 81.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":22.857142857142858,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":32.7212806026365,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":22.857142857142858,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 32,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 32.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":75.60975609756098,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":17.983333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.60975609756098,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":76.3157894736842,"measurements":[{"observationId":"vals-gpqa:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":91.414,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.3157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 739.636. Standardfehler: 1.406. Kosten je Aufgabe in USD: 0.025366. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Latency in seconds: 739.636. Standard error: 1.406. Cost per task in USD: 0.025366. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":56.09756097560975,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":37.2103799814643,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 37,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 37.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":5.555555555555555,"measurements":[{"observationId":"lmarena:webdev:overall:grok-4.3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1357.2716874069986,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":5.555555555555555,"participants":37,"methodology":{"de":"Arena-Bewertung aus 15.006 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: xAI. Modellstand: grok-4.3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 15,006 published pairwise votes.. Source model ID: lmarena/grok-4.3. Source model name: Grok 4.3. Provider reported by source: xAI. Model snapshot: grok-4.3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 15.006 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 15,006 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 15.006 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 15,006 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"lmarena:search:overall:grok-4.3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1165.805999435794,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":7.6923076923076925,"participants":14,"methodology":{"de":"Arena-Bewertung aus 128.393 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: xAI. Modellstand: grok-4.3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 128,393 published pairwise votes.. Source model ID: lmarena/grok-4.3. Source model name: Grok 4.3. Provider reported by source: xAI. Model snapshot: grok-4.3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 128.393 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 128,393 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 128.393 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 128,393 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":8.333333333333334,"measurements":[{"observationId":"lmarena:overall:grok-4.3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1441.5482021486605,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":8.333333333333334,"participants":37,"methodology":{"de":"Arena-Bewertung aus 71.387 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: xAI. Modellstand: grok-4.3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 71,387 published pairwise votes.. Source model ID: lmarena/grok-4.3. Source model name: Grok 4.3. Provider reported by source: xAI. Model snapshot: grok-4.3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 71.387 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 71,387 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 71.387 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 71,387 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"lmarena:vision_style_control:overall:grok-4.3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1242.51144955053,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":13.333333333333334,"participants":31,"methodology":{"de":"Arena-Bewertung aus 27.037 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: xAI. Modellstand: grok-4.3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 27,037 published pairwise votes.. Source model ID: lmarena/grok-4.3. Source model name: Grok 4.3. Provider reported by source: xAI. Model snapshot: grok-4.3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 27.037 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 27,037 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 27.037 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 27,037 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":51.08282674772036,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":78.09248554913299,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.357142857142854,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 78,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 78.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.064,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.808510638297875,"participants":48,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 150.948. Standardfehler: 0.902. Kosten je Aufgabe in USD: 0.02182. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Latency in seconds: 150.948. Standard error: 0.902. Cost per task in USD: 0.02182. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":31.35593220338983,"measurements":[{"observationId":"vals-swebench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":71.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.35593220338983,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 117.767. Standardfehler: 2.023. Kosten je Aufgabe in USD: 0.22259. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 117.767. Standard error: 2.023. Cost per task in USD: 0.22259. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":23.526422764227647,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:948892b5-db03-4118-a4a8-ccd51ed871ea","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.3 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/948892b5-db03-4118-a4a8-ccd51ed871ea/grok-4-3/grok-4-3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.3 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.3 (High) (Release grok-4-3, Zeile grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.3 (High) (release grok-4-3, row grok-4-3, UUID 948892b5-db03-4118-a4a8-ccd51ed871ea, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":43.446,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.76190476190476,"participants":43,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1954.824. Standardfehler: 4.406. Kosten je Aufgabe in USD: 1.115571. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1954.824. Standard error: 4.406. Cost per task in USD: 1.115571. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":41.948,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.821428571428571,"participants":57,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 470.713. Standardfehler: 2.622. Kosten je Aufgabe in USD: 0.982203. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 470.713. Standard error: 2.622. Cost per task in USD: 0.982203. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":14.516129032258064,"measurements":[{"observationId":"vals-code-migration:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":6.795,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.516129032258064,"participants":63,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 180.478. Standardfehler: 1.195. Kosten je Aufgabe in USD: 0.416921. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 180.478. Standard error: 1.195. Cost per task in USD: 0.416921. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":93.24324324324324,"measurements":[{"observationId":"vals-corp-fin-v2:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":68.532,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.24324324324324,"participants":75,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 36.444. Standardfehler: 0.915. Kosten je Aufgabe in USD: 0.134161. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.7. Top P: 0.95. Latency in seconds: 36.444. Standard error: 0.915. Cost per task in USD: 0.134161. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":8.474576271186441,"measurements":[{"observationId":"vals-emb:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":18.394,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.474576271186441,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 271.547. Standardfehler: 1.667. Kosten je Aufgabe in USD: 0.634124. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 271.547. Standard error: 1.667. Cost per task in USD: 0.634124. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"vals-finance-agent-v2:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":37.728,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 615.499. Standardfehler: 0.373. Kosten je Aufgabe in USD: 0.848345. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 615.499. Standard error: 0.373. Cost per task in USD: 0.848345. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":66.23376623376623,"measurements":[{"observationId":"vals-lcb:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":84.494,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.23376623376623,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 3248.962. Standardfehler: 1.024. Kosten je Aufgabe in USD: 0.020464. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 3248.962. Standard error: 1.024. Cost per task in USD: 0.020464. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":24.59016393442623,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.59016393442623,"participants":62,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 168.721. Standardfehler: 0. Kosten je Aufgabe in USD: 0.389144. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 168.721. Standard error: 0. Cost per task in USD: 0.389144. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":71.7948717948718,"measurements":[{"observationId":"vals-legal-bench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.458,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.7948717948718,"participants":79,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 348.006. Standardfehler: 0.424. Kosten je Aufgabe in USD: 0.003055. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Top P: 0.95. Latency in seconds: 348.006. Standard error: 0.424. Cost per task in USD: 0.003055. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":19.166666666666668,"measurements":[{"observationId":"vals-legal-research:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":15.385,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.166666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 84.583. Standardfehler: 2.508. Kosten je Aufgabe in USD: 0.203143. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 84.583. Standard error: 2.508. Cost per task in USD: 0.203143. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":22.388059701492537,"measurements":[{"observationId":"vals-medcode:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":38.068,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.388059701492537,"participants":68,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 43.924. Standardfehler: 2.081. Kosten je Aufgabe in USD: 0.022202. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 43.924. Standard error: 2.081. Cost per task in USD: 0.022202. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":22.058823529411764,"measurements":[{"observationId":"vals-medscribe:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":74.399,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.058823529411764,"participants":69,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 100.457. Standardfehler: 2.019. Kosten je Aufgabe in USD: 0.015293. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 100.457. Standard error: 2.019. Cost per task in USD: 0.015293. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":53.246753246753244,"measurements":[{"observationId":"vals-mmlu-pro:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.838,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.246753246753244,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 637.088. Standardfehler: 0.33. Kosten je Aufgabe in USD: 0.009489. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 637.088. Standard error: 0.33. Cost per task in USD: 0.009489. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":3.8461538461538463,"measurements":[{"observationId":"vals-mortgage-tax:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":48.252,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.8461538461538463,"participants":53,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 443.266. Standardfehler: 0.973. Kosten je Aufgabe in USD: 0.012184. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 443.266. Standard error: 0.973. Cost per task in USD: 0.012184. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 236.134. Standardfehler: 0. Kosten je Aufgabe in USD: 0.355823. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 236.134. Standard error: 0. Cost per task in USD: 0.355823. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":0,"measurements":[{"observationId":"vals-sage:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":19.736,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":59,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 96.2. Standardfehler: 2.647. Kosten je Aufgabe in USD: 0.037978. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 96.2. Standard error: 2.647. Cost per task in USD: 0.037978. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":20.689655172413794,"measurements":[{"observationId":"vals-skillsbench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":40.638,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.689655172413794,"participants":30,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 277.173. Standardfehler: 4.615. Kosten je Aufgabe in USD: 0.419712. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 277.173. Standard error: 4.615. Cost per task in USD: 0.419712. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":5.2631578947368425,"measurements":[{"observationId":"vals-snap:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":51.691,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.2631578947368425,"participants":39,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 205.01. Standardfehler: 1.3. Kosten je Aufgabe in USD: 0.264337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 205.01. Standard error: 1.3. Cost per task in USD: 0.264337. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-tax-agent-bench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":6.179,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":56,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 97.519. Standardfehler: 1.732. Kosten je Aufgabe in USD: 0.116616. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 97.519. Standard error: 1.732. Cost per task in USD: 0.116616. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":26.31578947368421,"measurements":[{"observationId":"vals-tax-eval-v2:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":70.81,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.31578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 391.972. Standardfehler: 0.896. Kosten je Aufgabe in USD: 0.013656. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Top P: 0.95. Latency in seconds: 391.972. Standard error: 0.896. Cost per task in USD: 0.013656. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Top P: 0.95","en":"Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":23.61111111111111,"measurements":[{"observationId":"vals-vibe-code-bench:overall:grok/grok-4.3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":19.403,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.61111111111111,"participants":73,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.3. Name des Quellmodells: Grok 4.3. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 589.407. Standardfehler: 4.243. Kosten je Aufgabe in USD: 1.28472. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.3. Source model name: Grok 4.3. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 589.407. Standard error: 4.243. Cost per task in USD: 1.28472. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":27.83988763201561,"coveredFamilies":38,"totalFamilies":71,"sourceCount":3,"coverage":0.5196145124716554,"lowerBound":14.466009639175457,"upperBound":62.504558392009926,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.42222222222222233,"lowerBound":11.312645083751754,"upperBound":69.09042286152953,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":18.430221896203925,"upperBound":75.57307903906107,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":64.42825467215711,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":42.95216978143807,"upperBound":76.28550311477142,"provisional":true,"configurationsDiffer":true},"agentic":{"score":27.087748785359082,"coveredFamilies":7,"totalFamilies":12,"sourceCount":2,"coverage":0.5833333333333334,"lowerBound":15.801186791459465,"upperBound":57.46785345812613,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":21.3768115942029,"upperBound":81.3768115942029,"provisional":true,"configurationsDiffer":true},"finance":{"score":26.272889765307248,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":22.519619798834782,"upperBound":36.805334084549074,"provisional":true,"configurationsDiffer":true},"legal":{"score":38.517234131988225,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":28.88792559899117,"upperBound":53.887925598991174,"provisional":true,"configurationsDiffer":true},"health":{"score":22.22344161545215,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":14.815627743634765,"upperBound":48.1489610769681,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":32.20808004052685,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":16.104040020263426,"upperBound":66.10404002026343,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":7.692307692307692,"upperBound":7.692307692307692,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":2.6315789473684212,"upperBound":52.631578947368425,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":0,"upperBound":50,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-4.5","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"research:grok-4.5:10","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9777777777777777,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":45.45454545454545,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: f8Pac6hqib4t7x2s4HCRoT. Startzeit: 2026-07-08T23:54:22.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9777777777777777 und stderr=0.012743145865697103. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: f8Pac6hqib4t7x2s4HCRoT. Started at 2026-07-08T23:54:22.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9777777777777777 and stderr=0.012743145865697103. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":35.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":35,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 35,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 35.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1280.57,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":79.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 79,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 79.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":57.93486035084273,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 57,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 57.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":74.39024390243902,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":18.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.39024390243902,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 18,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 18.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":64.63414634146342,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":1384.26,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":25.316666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":19.047619047619047,"measurements":[{"observationId":"research:grok-4.5:8","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.35,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":19.047619047619047,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: jTuYE8VJTos73Kw77UFevq. Startzeit: 2026-10-13T13:14:23.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.35 und stderr=0.047937248544110224. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: jTuYE8VJTos73Kw77UFevq. Started at 2026-10-13T13:14:23.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.35 and stderr=0.047937248544110224. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"research:grok-4.5:9","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.483,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":33.333333333333336,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: 7HWW3sjRstKzb9JpcnvLaV. Startzeit: 2026-08-27T19:29:30.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.483 und stderr=0.015810153729833274. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: 7HWW3sjRstKzb9JpcnvLaV. Started at 2026-08-27T19:29:30.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.483 and stderr=0.015810153729833274. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":20.707070707070706,"measurements":[{"observationId":"research:grok-4.5:6","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.24390243902439024,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":27.77777777777778,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: 7vThbrtbczScg2UcsAYXtF. Startzeit: 2026-07-09T01:53:42.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.24390243902439024 und stderr=0.06789956540036612. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: 7vThbrtbczScg2UcsAYXtF. Started at 2026-07-09T01:53:42.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.24390243902439024 and stderr=0.06789956540036612. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"research:grok-4.5:5","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.5719298245614035,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":13.636363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: djbbEqvBW9qux3pZwcZN2i. Startzeit: 2026-07-09T01:53:42.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.5719298245614035 und stderr=0.029360921879029937. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: djbbEqvBW9qux3pZwcZN2i. Started at 2026-07-09T01:53:42.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.5719298245614035 and stderr=0.029360921879029937. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":73.38659147869674,"measurements":[{"observationId":"research:grok-4.5:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9343434343434344,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":61.904761904761905,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.5_high. Epoch-Modellbezeichnung: Grok 4.5. Anzeigenamenfeld: Grok 4.5 (high). Im Alias kodierte Reasoning-Stufe: high. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: 9kSgmUd9vh4CBXz3tJhctN. Startzeit: 2026-07-08T23:54:22.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9343434343434344 und stderr=0.014340395485738813. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: high.","en":"Independent evaluation. Epoch run alias: grok-4.5_high; Epoch model label: Grok 4.5; Display label field: Grok 4.5 (high); Reasoning setting encoded in the alias: high. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: 9kSgmUd9vh4CBXz3tJhctN. Started at 2026-07-08T23:54:22.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9343434343434344 and stderr=0.014340395485738813. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: high."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: high. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: high. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":92.929,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.86842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 217.858. Standardfehler: 1.288. Kosten je Aufgabe in USD: 0.044029. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 217.858. Standard error: 1.288. Cost per task in USD: 0.044029. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":75,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":42.678405931418,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 42,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 42.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":41.666666666666664,"measurements":[{"observationId":"lmarena:webdev:overall:grok-4.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1552.9312021719509,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":41.666666666666664,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.118 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: xAI. Modellstand: grok-4.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,118 published pairwise votes.. Source model ID: lmarena/grok-4.5. Source model name: Grok 4.5. Provider reported by source: xAI. Model snapshot: grok-4.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.118 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 11,118 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.118 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 11,118 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-search","area":"search","label":{"de":"LMArena Search","en":"LMArena Search"},"weight":0.07142857142857142,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"lmarena:search:overall:grok-4.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/search","benchmarkId":"lmarena-search","benchmarkVersion":"search","comparisonCohort":"lmarena-search:2026-10-13:overall","retrievedAt":"2026-10-15","value":1215.9370825422614,"unit":"rating","metric":"Search Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":46.15384615384615,"participants":14,"methodology":{"de":"Arena-Bewertung aus 76.458 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: xAI. Modellstand: grok-4.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Search. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Prüfsumme der Quelldaten: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 76,458 published pairwise votes.. Source model ID: lmarena/grok-4.5. Source model name: Grok 4.5. Provider reported by source: xAI. Model snapshot: grok-4.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Search. Source ID: lmarena. Evidence capture ID: lmarena-search-4589dee83b594891961ae5190c508a9288d2c2ae-698bdef0df20. Source payload hash: 698bdef0df20299667345d788e41f4f03e0be49e68ca51c67bb8953586b58478. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 76.458 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 76,458 published pairwise votes.. Harness: Arena Search"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 76.458 veröffentlichten Paarvergleichen.. Harness: Arena Search","en":"Arena rating from 76,458 published pairwise votes.. Harness: Arena Search"},"scoreComparisonSetup":{"publicationCohort":"lmarena-search:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Search","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":38.888888888888886,"measurements":[{"observationId":"lmarena:overall:grok-4.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1465.7273040319747,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":38.888888888888886,"participants":37,"methodology":{"de":"Arena-Bewertung aus 39.899 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: xAI. Modellstand: grok-4.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 39,899 published pairwise votes.. Source model ID: lmarena/grok-4.5. Source model name: Grok 4.5. Provider reported by source: xAI. Model snapshot: grok-4.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 39.899 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 39,899 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 39.899 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 39,899 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":60,"measurements":[{"observationId":"lmarena:vision_style_control:overall:grok-4.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1280.0166899379963,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":60,"participants":31,"methodology":{"de":"Arena-Bewertung aus 11.764 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: xAI. Modellstand: grok-4.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,764 published pairwise votes.. Source model ID: lmarena/grok-4.5. Source model name: Grok 4.5. Provider reported by source: xAI. Model snapshot: grok-4.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.764 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 11,764 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.764 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 11,764 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":40.86246200607903,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":80.4046242774566,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.21428571428571,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 80,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 80.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":61.792,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.51063829787234,"participants":48,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 714.648. Standardfehler: 1.168. Kosten je Aufgabe in USD: 0.030117. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 714.648. Standard error: 1.168. Cost per task in USD: 0.030117. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":83.05084745762711,"measurements":[{"observationId":"vals-swebench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":86.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.05084745762711,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 199.573. Standardfehler: 1.525. Kosten je Aufgabe in USD: 0.54034. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 199.573. Standard error: 1.525. Cost per task in USD: 0.54034. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":53.640526414916664,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:794f69b5-cede-482b-b1cc-d769478497cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":10.6060606060606,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":64.63414634146342,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.5 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 10,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/794f69b5-cede-482b-b1cc-d769478497cd/grok-4-5/grok-4-5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.5 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 10.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.5 (High) (Release grok-4-5, Zeile grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.5 (High) (release grok-4-5, row grok-4-5, UUID 794f69b5-cede-482b-b1cc-d769478497cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":67.79,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.67857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 624.415. Standardfehler: 4.416. Kosten je Aufgabe in USD: 0.283943. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 624.415. Standard error: 4.416. Cost per task in USD: 0.283943. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":8.586,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.62162162162162,"participants":38,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2022.621. Standardfehler: 1.336. Kosten je Aufgabe in USD: 5.51917. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2022.621. Standard error: 1.336. Cost per task in USD: 5.51917. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":59.67741935483871,"measurements":[{"observationId":"vals-code-migration:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":36.596,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.67741935483871,"participants":63,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1408.81. Standardfehler: 4.141. Kosten je Aufgabe in USD: 3.369806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1408.81. Standard error: 4.141. Cost per task in USD: 3.369806. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":87.83783783783784,"measurements":[{"observationId":"vals-corp-fin-v2:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":67.405,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.83783783783784,"participants":75,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 91.881. Standardfehler: 0.924. Kosten je Aufgabe in USD: 0.221291. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 91.881. Standard error: 0.924. Cost per task in USD: 0.221291. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-cyberbench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":71.012,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 848.118. Standardfehler: 5.793. Kosten je Aufgabe in USD: 2.949433. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 848.118. Standard error: 5.793. Cost per task in USD: 2.949433. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":33.898305084745765,"measurements":[{"observationId":"vals-emb:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":52.911,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.898305084745765,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1107.838. Standardfehler: 3.111. Kosten je Aufgabe in USD: 1.55492. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1107.838. Standard error: 3.111. Cost per task in USD: 1.55492. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":41.666666666666664,"measurements":[{"observationId":"vals-finance-agent-v2:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":48.349,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 289.014. Standardfehler: 0.458. Kosten je Aufgabe in USD: 1.156495. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 289.014. Standard error: 0.458. Cost per task in USD: 1.156495. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-ioi:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":40.556,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 5194.818. Standardfehler: 3.951. Kosten je Aufgabe in USD: 5.637814. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 5194.818. Standard error: 3.951. Cost per task in USD: 5.637814. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":84.41558441558442,"measurements":[{"observationId":"vals-lcb:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.351,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.41558441558442,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 589.356. Standardfehler: 0.961. Kosten je Aufgabe in USD: 0.045235. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 589.356. Standard error: 0.961. Cost per task in USD: 0.045235. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":90.98360655737704,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":12.917,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.98360655737704,"participants":62,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 614.705. Standardfehler: 2.534. Kosten je Aufgabe in USD: 2.01955. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 614.705. Standard error: 2.534. Cost per task in USD: 2.01955. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":80.76923076923077,"measurements":[{"observationId":"vals-legal-bench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":85.965,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.76923076923077,"participants":79,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 67.875. Standardfehler: 0.413. Kosten je Aufgabe in USD: 0.004535. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 67.875. Standard error: 0.413. Cost per task in USD: 0.004535. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":56.666666666666664,"measurements":[{"observationId":"vals-legal-research:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":37.981,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.666666666666664,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 953.767. Standardfehler: 3.373. Kosten je Aufgabe in USD: 1.086814. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 953.767. Standard error: 3.373. Cost per task in USD: 1.086814. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":49.25373134328358,"measurements":[{"observationId":"vals-medcode:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":43.291,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.25373134328358,"participants":68,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 59.637. Standardfehler: 2.313. Kosten je Aufgabe in USD: 0.048461. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 59.637. Standard error: 2.313. Cost per task in USD: 0.048461. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":80.88235294117646,"measurements":[{"observationId":"vals-medscribe:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":86.884,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.88235294117646,"participants":69,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 704.513. Standardfehler: 1.944. Kosten je Aufgabe in USD: 0.033208. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 704.513. Standard error: 1.944. Cost per task in USD: 0.033208. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":84.41558441558442,"measurements":[{"observationId":"vals-mmlu-pro:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.216,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.41558441558442,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 357.751. Standardfehler: 0.31. Kosten je Aufgabe in USD: 0.014974. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 357.751. Standard error: 0.31. Cost per task in USD: 0.014974. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"vals-mortgage-tax:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":61.804,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":53,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 117.461. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.014998. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 117.461. Standard error: 0.968. Cost per task in USD: 0.014998. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1139.043. Standardfehler: 0. Kosten je Aufgabe in USD: 4.124197. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1139.043. Standard error: 0. Cost per task in USD: 4.124197. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":37.17948717948718,"measurements":[{"observationId":"vals-proof-bench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":31,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.17948717948718,"participants":40,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 668.86. Standardfehler: 4.648. Kosten je Aufgabe in USD: 0.573991. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 668.86. Standard error: 4.648. Cost per task in USD: 0.573991. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":13.793103448275861,"measurements":[{"observationId":"vals-sage:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":34.967,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.793103448275861,"participants":59,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 169.791. Standardfehler: 3.316. Kosten je Aufgabe in USD: 0.115513. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 169.791. Standard error: 3.316. Cost per task in USD: 0.115513. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":96.55172413793103,"measurements":[{"observationId":"vals-skillsbench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":66.034,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.55172413793103,"participants":30,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 135.652. Standardfehler: 4.596. Kosten je Aufgabe in USD: 0.578155. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 135.652. Standard error: 4.596. Cost per task in USD: 0.578155. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":20.833333333333332,"measurements":[{"observationId":"vals-sre-bench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0.763,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.833333333333332,"participants":25,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 3939.029. Standardfehler: 0.539. Kosten je Aufgabe in USD: 13.415223. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 3939.029. Standard error: 0.539. Cost per task in USD: 13.415223. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-tax-agent-bench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":24.855,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":56,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 283.566. Standardfehler: 3.118. Kosten je Aufgabe in USD: 0.473604. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 283.566. Standard error: 3.118. Cost per task in USD: 0.473604. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":36.8421052631579,"measurements":[{"observationId":"vals-tax-eval-v2:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.668,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.8421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 195.458. Standardfehler: 0.892. Kosten je Aufgabe in USD: 0.019211. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 195.458. Standard error: 0.892. Cost per task in USD: 0.019211. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":61.111111111111114,"measurements":[{"observationId":"vals-vibe-code-bench:overall:grok/grok-4.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":69.002,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.111111111111114,"participants":73,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.5. Name des Quellmodells: Grok 4.5. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 830.057. Standardfehler: 4.48. Kosten je Aufgabe in USD: 4.102194. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.5. Source model name: Grok 4.5. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 830.057. Standard error: 4.48. Cost per task in USD: 4.102194. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":54.01339114520675,"coveredFamilies":42,"totalFamilies":71,"sourceCount":4,"coverage":0.6301587301587301,"lowerBound":34.03700997563028,"upperBound":71.02113695975727,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.48888888888888904,"lowerBound":25.051217203753154,"upperBound":76.16232831486425,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":58.41356439880188,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":37.55157711351549,"upperBound":73.26586282780121,"provisional":true,"configurationsDiffer":true},"math":{"score":34.44703444703445,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":20.66822066822067,"upperBound":60.66822066822066,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":67.29841973744414,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":67.29841973744414,"upperBound":67.29841973744414,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":28.892682189072346,"upperBound":87.22601552240567,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":11.944444444444443,"upperBound":71.94444444444444,"provisional":true,"configurationsDiffer":true},"finance":{"score":46.3112154124643,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":39.69532749639797,"upperBound":53.981041782112264,"provisional":true,"configurationsDiffer":true},"legal":{"score":76.13983466442482,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":57.10487599831862,"upperBound":82.10487599831862,"provisional":true,"configurationsDiffer":true},"health":{"score":65.06804214223003,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":43.378694761486685,"upperBound":76.71202809482003,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":66.66666666666667,"upperBound":66.66666666666667,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":50.431231003039514,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":25.215615501519757,"upperBound":75.21561550151975,"provisional":true,"configurationsDiffer":true},"search":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":46.15384615384615,"upperBound":46.15384615384615,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":6.896551724137931,"upperBound":56.89655172413793,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-4.6","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"research:grok-4.6:19","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9916666666666667,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":72.72727272727273,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: 7crLvnfQc987dC6mr3iGbf. Startzeit: 2026-08-14T13:47:42.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9916666666666667 und stderr=0.004700633956814717. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: 7crLvnfQc987dC6mr3iGbf. Started at 2026-08-14T13:47:42.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9916666666666667 and stderr=0.004700633956814717. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":48.68421052631579,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:c8adc5cf-fd5a-407b-af51-dc3bede3e49c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":41.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.68421052631579,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/c8adc5cf-fd5a-407b-af51-dc3bede3e49c/grok-4-6/grok-4-6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 41,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/c8adc5cf-fd5a-407b-af51-dc3bede3e49c/grok-4-6/grok-4-6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 41.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":89.02439024390245,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1544.92,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":81,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 81,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 81.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":66.99736836183247,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 67,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 67.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":17.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 17,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 17.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1645.38,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":90.2439024390244,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":29.316666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":90.2439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"research:grok-4.6:15","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.33,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":14.285714285714286,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: NiuExVccWkWbWqi6ex6vvf. Startzeit: 2026-10-13T13:14:03.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.33 und stderr=0.04725815626252608. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: NiuExVccWkWbWqi6ex6vvf. Started at 2026-10-13T13:14:03.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.33 and stderr=0.04725815626252608. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":38.888888888888886,"measurements":[{"observationId":"research:grok-4.6:17","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.489,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":38.888888888888886,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: dKCWFaoGCWJTnD5CPWQGru. Startzeit: 2026-08-27T19:29:19.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.489 und stderr=0.015815471195292575. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: dKCWFaoGCWJTnD5CPWQGru. Started at 2026-08-27T19:29:19.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.489 and stderr=0.015815471195292575. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":45.95959595959596,"measurements":[{"observationId":"research:grok-4.6:11","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.3170731707317073,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":55.55555555555556,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: bg3JRdd33iWfvRoGSvyXXE. Startzeit: 2026-08-14T15:46:50.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.3170731707317073 und stderr=0.07357611282438223. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: bg3JRdd33iWfvRoGSvyXXE. Started at 2026-08-14T15:46:50.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.3170731707317073 and stderr=0.07357611282438223. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"research:grok-4.6:10","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.6596491228070176,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":36.36363636363637,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: Zb2AKwa942D7dBTTcebAba. Startzeit: 2026-08-14T15:46:50.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.6596491228070176 und stderr=0.028116467881852295. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: Zb2AKwa942D7dBTTcebAba. Started at 2026-08-14T15:46:50.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.6596491228070176 and stderr=0.028116467881852295. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":77.25563909774436,"measurements":[{"observationId":"research:grok-4.6:13","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9318181818181818,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":57.142857142857146,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.6_xhigh. Epoch-Modellbezeichnung: Grok 4.6. Anzeigenamenfeld: Grok 4.6 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: 9YJf2nviwjM6WVhw8qeJta. Startzeit: 2026-08-14T13:47:42.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9318181818181818 und stderr=0.015204299687218694. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.6_xhigh; Epoch model label: Grok 4.6; Display label field: Grok 4.6 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: 9YJf2nviwjM6WVhw8qeJta. Started at 2026-08-14T13:47:42.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9318181818181818 and stderr=0.015204299687218694. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":94.697,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 209.056. Standardfehler: 1.125. Kosten je Aufgabe in USD: 0.045365. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 209.056. Standard error: 1.125. Cost per task in USD: 0.045365. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":44.0685820203893,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 44,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 44.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":58.333333333333336,"measurements":[{"observationId":"lmarena:webdev:overall:grok-4.6-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1617.4015018305167,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":58.333333333333336,"participants":37,"methodology":{"de":"Arena-Bewertung aus 9.774 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.6-high. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: xAI. Modellstand: grok-4.6-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 9,774 published pairwise votes.. Source model ID: lmarena/grok-4.6-high. Source model name: Grok 4.6. Provider reported by source: xAI. Model snapshot: grok-4.6-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 9.774 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 9,774 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 9.774 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: high","en":"Arena rating from 9,774 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":25,"measurements":[{"observationId":"lmarena:overall:grok-4.6-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1454.9756332800775,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":25,"participants":37,"methodology":{"de":"Arena-Bewertung aus 28.124 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.6-high. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: xAI. Modellstand: grok-4.6-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 28,124 published pairwise votes.. Source model ID: lmarena/grok-4.6-high. Source model name: Grok 4.6. Provider reported by source: xAI. Model snapshot: grok-4.6-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 28.124 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 28,124 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 28.124 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: high","en":"Arena rating from 28,124 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":30,"measurements":[{"observationId":"lmarena:vision_style_control:overall:grok-4.6-high","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1261.8713594392048,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":30,"participants":31,"methodology":{"de":"Arena-Bewertung aus 6.495 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.6-high. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: xAI. Modellstand: grok-4.6-high. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: high. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 6,495 published pairwise votes.. Source model ID: lmarena/grok-4.6-high. Source model name: Grok 4.6. Provider reported by source: xAI. Model snapshot: grok-4.6-high. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: high. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 6.495 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 6,495 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 6.495 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: high","en":"Arena rating from 6,495 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":96.61016949152543,"measurements":[{"observationId":"vals-swebench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":95.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.61016949152543,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 604.494. Standardfehler: 0.918. Kosten je Aufgabe in USD: 0.784684. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 604.494. Standard error: 0.918. Cost per task in USD: 0.784684. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":70.54736792541671,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":17.171717171717198,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 17,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3/grok-4-6/grok-4-6-xhigh. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 17.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (Xhigh) (Release grok-4-6, Zeile grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.6 (Xhigh) (release grok-4-6, row grok-4-6-xhigh, UUID d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":78.277,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.35714285714286,"participants":57,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 642.908. Standardfehler: 2.085. Kosten je Aufgabe in USD: 0.453642. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 642.908. Standard error: 2.085. Cost per task in USD: 0.453642. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":17.172,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.891891891891895,"participants":38,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1765.468. Standardfehler: 1.336. Kosten je Aufgabe in USD: 5.062449. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1765.468. Standard error: 1.336. Cost per task in USD: 5.062449. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":50.21037868162693,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:c8adc5cf-fd5a-407b-af51-dc3bede3e49c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":6.19047619047619,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.130434782608695,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/c8adc5cf-fd5a-407b-af51-dc3bede3e49c/grok-4-6/grok-4-6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.6 (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 6,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/c8adc5cf-fd5a-407b-af51-dc3bede3e49c/grok-4-6/grok-4-6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.6 (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 6.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.6 (High) (Release grok-4-6, Zeile grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: Grok 4.6 (High) (release grok-4-6, row grok-4-6, UUID c8adc5cf-fd5a-407b-af51-dc3bede3e49c, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":11.429,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.29032258064516,"participants":32,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 4696.051. Standardfehler: 3.83. Kosten je Aufgabe in USD: 4.815933. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 4696.051. Standard error: 3.83. Cost per task in USD: 4.815933. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":57.89473684210526,"measurements":[{"observationId":"vals-biomysterybench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":72.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.89473684210526,"participants":20,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 858.019. Standardfehler: 0. Kosten je Aufgabe in USD: 1.226606. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 858.019. Standard error: 0. Cost per task in USD: 1.226606. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":74.19354838709677,"measurements":[{"observationId":"vals-code-migration:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.572,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.19354838709677,"participants":63,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 5429.239. Standardfehler: 4.479. Kosten je Aufgabe in USD: 14.575491. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 5429.239. Standard error: 4.479. Cost per task in USD: 14.575491. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":81.08108108108108,"measurements":[{"observationId":"vals-corp-fin-v2:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.161,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.08108108108108,"participants":75,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 24.256. Standardfehler: 0.932. Kosten je Aufgabe in USD: 0.165532. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 24.256. Standard error: 0.932. Cost per task in USD: 0.165532. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":48.484848484848484,"measurements":[{"observationId":"vals-cyberbench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":67.678,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.484848484848484,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2212.209. Standardfehler: 5.866. Kosten je Aufgabe in USD: 4.06856. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2212.209. Standard error: 5.866. Cost per task in USD: 4.06856. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":59.32203389830509,"measurements":[{"observationId":"vals-emb:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":62.727,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.32203389830509,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 3178.533. Standardfehler: 3.085. Kosten je Aufgabe in USD: 3.056382. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 3178.533. Standard error: 3.085. Cost per task in USD: 3.056382. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":65,"measurements":[{"observationId":"vals-finance-agent-v2:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.682,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1000.937. Standardfehler: 0.67. Kosten je Aufgabe in USD: 1.657732. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1000.937. Standard error: 0.67. Cost per task in USD: 1.657732. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-ioi:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":47.611,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 14233.553. Standardfehler: 2.091. Kosten je Aufgabe in USD: 8.183386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 14233.553. Standard error: 2.091. Cost per task in USD: 8.183386. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":92.20779220779221,"measurements":[{"observationId":"vals-lcb:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":88.224,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.20779220779221,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 118.4. Standardfehler: 0.941. Kosten je Aufgabe in USD: 0.039923. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 118.4. Standard error: 0.941. Cost per task in USD: 0.039923. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":94.26229508196721,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":15.833,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.26229508196721,"participants":62,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2705.048. Standardfehler: 2.645. Kosten je Aufgabe in USD: 4.008154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2705.048. Standard error: 2.645. Cost per task in USD: 4.008154. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":85.8974358974359,"measurements":[{"observationId":"vals-legal-bench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.307,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.8974358974359,"participants":79,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 27.42. Standardfehler: 0.418. Kosten je Aufgabe in USD: 0.01457. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 27.42. Standard error: 0.418. Cost per task in USD: 0.01457. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":48.077,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1504.077. Standardfehler: 3.473. Kosten je Aufgabe in USD: 1.530204. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1504.077. Standard error: 3.473. Cost per task in USD: 1.530204. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":56.71641791044776,"measurements":[{"observationId":"vals-medcode:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":44.712,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.71641791044776,"participants":68,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 127.284. Standardfehler: 2.256. Kosten je Aufgabe in USD: 0.050335. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 127.284. Standard error: 2.256. Cost per task in USD: 0.050335. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":76.47058823529412,"measurements":[{"observationId":"vals-medscribe:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":86.534,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.47058823529412,"participants":69,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 70.944. Standardfehler: 1.956. Kosten je Aufgabe in USD: 0.03719. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 70.944. Standard error: 1.956. Cost per task in USD: 0.03719. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":88.31168831168831,"measurements":[{"observationId":"vals-mmlu-pro:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.31168831168831,"participants":78,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 54.975. Standardfehler: 0.305. Kosten je Aufgabe in USD: 0.015391. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 54.975. Standard error: 0.305. Cost per task in USD: 0.015391. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":32.69230769230769,"measurements":[{"observationId":"vals-mortgage-tax:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":64.189,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.69230769230769,"participants":53,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 8.548. Standardfehler: 0.952. Kosten je Aufgabe in USD: 0.013712. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 8.548. Standard error: 0.952. Cost per task in USD: 0.013712. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":50,"measurements":[{"observationId":"vals-mysterymechanism:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":30.631,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":21,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1674.514. Standardfehler: 3.101. Kosten je Aufgabe in USD: 1.309658. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1674.514. Standard error: 3.101. Cost per task in USD: 1.309658. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":73.46938775510205,"measurements":[{"observationId":"vals-programbench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":1,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.46938775510205,"participants":50,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 16044.462. Standardfehler: 0.705. Kosten je Aufgabe in USD: 16.759019. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 16044.462. Standard error: 0.705. Cost per task in USD: 16.759019. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"vals-proof-bench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":51,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":40,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 861.898. Standardfehler: 5.024. Kosten je Aufgabe in USD: 0.759043. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 861.898. Standard error: 5.024. Cost per task in USD: 0.759043. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":5.172413793103448,"measurements":[{"observationId":"vals-sage:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":28.9,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.172413793103448,"participants":59,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 202.474. Standardfehler: 3.076. Kosten je Aufgabe in USD: 0.109757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 202.474. Standard error: 3.076. Cost per task in USD: 0.109757. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":65.51724137931035,"measurements":[{"observationId":"vals-skillsbench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":55.766,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.51724137931035,"participants":30,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 808.117. Standardfehler: 4.844. Kosten je Aufgabe in USD: 0.921477. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 808.117. Standard error: 4.844. Cost per task in USD: 0.921477. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":71.05263157894737,"measurements":[{"observationId":"vals-snap:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":66.847,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.05263157894737,"participants":39,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1552.915. Standardfehler: 1.225. Kosten je Aufgabe in USD: 0.91559. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1552.915. Standard error: 1.225. Cost per task in USD: 0.91559. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":80,"measurements":[{"observationId":"vals-tax-agent-bench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":33.141,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80,"participants":56,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 657.844. Standardfehler: 3.396. Kosten je Aufgabe in USD: 0.978328. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 657.844. Standard error: 3.396. Cost per task in USD: 0.978328. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":29.605263157894736,"measurements":[{"observationId":"vals-tax-eval-v2:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.096,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.605263157894736,"participants":77,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 47.57. Standardfehler: 0.899. Kosten je Aufgabe in USD: 0.017298. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 47.57. Standard error: 0.899. Cost per task in USD: 0.017298. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":28.571428571428573,"measurements":[{"observationId":"vals-vcb-1-100:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":14.75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.571428571428573,"participants":22,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 7851.405. Standardfehler: 3.17. Kosten je Aufgabe in USD: 26.271157. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 7851.405. Standard error: 3.17. Cost per task in USD: 26.271157. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":68.05555555555556,"measurements":[{"observationId":"vals-vibe-code-bench:overall:grok/grok-4.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":76.239,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.05555555555556,"participants":73,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.6. Name des Quellmodells: Grok 4.6. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1530.145. Standardfehler: 3.824. Kosten je Aufgabe in USD: 4.875513. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.6. Source model name: Grok 4.6. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1530.145. Standard error: 3.824. Cost per task in USD: 4.875513. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":59.233011498265746,"coveredFamilies":44,"totalFamilies":71,"sourceCount":4,"coverage":0.6003968253968255,"lowerBound":35.563312062252415,"upperBound":75.52362952256988,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":62.728855268808445,"coveredFamilies":10,"totalFamilies":15,"sourceCount":3,"coverage":0.6222222222222223,"lowerBound":39.03128772281415,"upperBound":76.80906550059191,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":61.39943354320762,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":39.47106442063347,"upperBound":75.18535013491919,"provisional":true,"configurationsDiffer":true},"math":{"score":54.94690494690495,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":32.96814296814297,"upperBound":72.96814296814296,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":72.48149321320054,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":72.48149321320054,"upperBound":72.48149321320054,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":32.48426733359893,"upperBound":90.81760066693226,"provisional":true,"configurationsDiffer":true},"general":{"score":44.29824561403509,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":26.578947368421055,"upperBound":66.57894736842104,"provisional":true,"configurationsDiffer":true},"finance":{"score":57.95011430493143,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":49.67152654708408,"upperBound":63.95724083279837,"provisional":true,"configurationsDiffer":true},"legal":{"score":88.94213254868993,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":66.70659941151744,"upperBound":91.70659941151744,"provisional":true,"configurationsDiffer":true},"health":{"score":66.59350307287094,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":44.3956687152473,"upperBound":77.72900204858064,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":48.484848484848484,"upperBound":48.484848484848484,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":7.5,"upperBound":82.5,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":35.526315789473685,"upperBound":85.52631578947368,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":2.586206896551724,"upperBound":52.58620689655172,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-4.7","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":50,"measurements":[{"observationId":"research:grok-4.7:12","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9805555555555555,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":50,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: Jhgwm7QpspGcvm7DUVqUDg. Startzeit: 2026-09-22T21:49:22.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9805555555555555 und stderr=0.014299486306926387. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: Jhgwm7QpspGcvm7DUVqUDg. Started at 2026-09-22T21:49:22.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9805555555555555 and stderr=0.014299486306926387. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":84.21052631578948,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":52.5,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.21052631578948,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 52,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 52.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1641.75,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":44.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":76.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":44.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 76,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 76.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":65.55835148969457,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 65,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 65.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":33.834199999999996,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.333333333333334,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 33,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 33.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":20,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 20,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 20.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":1711.07,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":60,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":42.0809792843691,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 42,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 42.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":32.03333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":42.857142857142854,"measurements":[{"observationId":"research:grok-4.7:10","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.41,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":42.857142857142854,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: L5Ck7pMy8bVmVMMLwxJN7f. Startzeit: 2026-10-13T12:39:21.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.41 und stderr=0.04943110704237103. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: L5Ck7pMy8bVmVMMLwxJN7f. Started at 2026-10-13T12:39:21.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.41 and stderr=0.04943110704237103. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":50,"measurements":[{"observationId":"research:grok-4.7:11","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.56,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":50,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: 6izDCw5ygCLZJEKPS8jV5T. Startzeit: 2026-09-22T21:49:22.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.56 und stderr=0.015704987954361718. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: 6izDCw5ygCLZJEKPS8jV5T. Started at 2026-09-22T21:49:22.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.56 and stderr=0.015704987954361718. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":6.4393939393939394,"measurements":[{"observationId":"research:grok-4.7:8","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.17073170731707318,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":8.333333333333334,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: Ymgj5VeEceBbzsLq6xEZRi. Startzeit: 2026-09-22T21:49:22.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.17073170731707318 und stderr=0.05949419959829495. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: Ymgj5VeEceBbzsLq6xEZRi. Started at 2026-09-22T21:49:22.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.17073170731707318 and stderr=0.05949419959829495. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"research:grok-4.7:7","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.5298245614035088,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":4.545454545454546,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: F4unBbW5M4VJogzCq4vGAb. Startzeit: 2026-09-22T21:49:22.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.5298245614035088 und stderr=0.029616712041900365. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: F4unBbW5M4VJogzCq4vGAb. Started at 2026-09-22T21:49:22.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.5298245614035088 and stderr=0.029616712041900365. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":47.61904761904762,"measurements":[{"observationId":"research:grok-4.7:9","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.92739898989899,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":47.61904761904762,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: grok-4.7_xhigh. Epoch-Modellbezeichnung: Grok 4.7. Anzeigenamenfeld: Grok 4.7 (xhigh). Im Alias kodierte Reasoning-Stufe: xhigh. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: hxAhYJfXdPNanxatXpG5hC. Startzeit: 2026-09-22T21:49:22.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.92739898989899 und stderr=0.015505824295534454. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: xhigh.","en":"Independent evaluation. Epoch run alias: grok-4.7_xhigh; Epoch model label: Grok 4.7; Display label field: Grok 4.7 (xhigh); Reasoning setting encoded in the alias: xhigh. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: hxAhYJfXdPNanxatXpG5hC. Started at 2026-09-22T21:49:22.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.92739898989899 and stderr=0.015505824295534454. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: xhigh."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: xhigh. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: xhigh. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":43.1417979610751,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 43,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 43.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":69.44444444444444,"measurements":[{"observationId":"lmarena:webdev:overall:grok-4.7-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1640.148880352992,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":69.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 4.633 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.7-xhigh. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: xAI. Modellstand: grok-4.7-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 4,633 published pairwise votes.. Source model ID: lmarena/grok-4.7-xhigh. Source model name: Grok 4.7. Provider reported by source: xAI. Model snapshot: grok-4.7-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 4.633 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh","en":"Arena rating from 4,633 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 4.633 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: xhigh","en":"Arena rating from 4,633 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":13.88888888888889,"measurements":[{"observationId":"lmarena:overall:grok-4.7-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1443.8399759103029,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":13.88888888888889,"participants":37,"methodology":{"de":"Arena-Bewertung aus 10.768 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.7-xhigh. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: xAI. Modellstand: grok-4.7-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,768 published pairwise votes.. Source model ID: lmarena/grok-4.7-xhigh. Source model name: Grok 4.7. Provider reported by source: xAI. Model snapshot: grok-4.7-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.768 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 10,768 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.768 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 10,768 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"lmarena:vision_style_control:overall:grok-4.7-xhigh","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1247.5489578886074,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":16.666666666666668,"participants":31,"methodology":{"de":"Arena-Bewertung aus 2.942 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/grok-4.7-xhigh. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: xAI. Modellstand: grok-4.7-xhigh. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,942 published pairwise votes.. Source model ID: lmarena/grok-4.7-xhigh. Source model name: Grok 4.7. Provider reported by source: xAI. Model snapshot: grok-4.7-xhigh. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: xhigh. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.942 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 2,942 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.942 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: xhigh","en":"Arena rating from 2,942 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: xhigh"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":57.142857142857146,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=Low","retrievedAt":"2026-10-19","value":78.150289017341,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.142857142857146,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Low) (Release grok-4-7, Zeile grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/208c16f0-8c5b-4524-ac8a-ac74c02e80a9/grok-4-7/grok-4-7-low. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Low). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Low. Angezeigter Wert: 78,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Low) (release grok-4-7, row grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/208c16f0-8c5b-4524-ac8a-ac74c02e80a9/grok-4-7/grok-4-7-low. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Low). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Low. Score display: 78.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Low) (Release grok-4-7, Zeile grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Low","en":"Exact AA configuration: Grok 4.7 (Low) (release grok-4-7, row grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Low"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Low) (Release grok-4-7, Zeile grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Low","en":"Exact AA configuration: Grok 4.7 (Low) (release grok-4-7, row grok-4-7-low, UUID 208c16f0-8c5b-4524-ac8a-ac74c02e80a9, reasoning=true, source reasoning-token configuration budget=0, suffix=Low). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Low"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Low","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":73.39886053300687,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":25.7575757575758,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 25,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 25.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":73.408,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.07142857142857,"participants":57,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 797.952. Standardfehler: 1.498. Kosten je Aufgabe in USD: 1.076601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 797.952. Standard error: 1.498. Cost per task in USD: 1.076601. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":28.788,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.45945945945946,"participants":38,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 3007.523. Standardfehler: 2.314. Kosten je Aufgabe in USD: 18.011582. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 3007.523. Standard error: 2.314. Cost per task in USD: 18.011582. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":41.54978962131837,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:59627802-1546-422c-9a65-65b61c022d36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=0:suffix=Xhigh","retrievedAt":"2026-10-19","value":4.76190476190476,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.26086956521739,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Modellstand: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 4,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/59627802-1546-422c-9a65-65b61c022d36/grok-4-7/grok-4-7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Model snapshot: Grok 4.7 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 4.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Grok 4.7 (Xhigh) (Release grok-4-7, Zeile grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Grok 4.7 (Xhigh) (release grok-4-7, row grok-4-7, UUID 59627802-1546-422c-9a65-65b61c022d36, reasoning=true, source reasoning-token configuration budget=0, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":10,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.83870967741935,"participants":32,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 5553.191. Standardfehler: 3.612. Kosten je Aufgabe in USD: 17.040638. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 5553.191. Standard error: 3.612. Cost per task in USD: 17.040638. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":42.10526315789474,"measurements":[{"observationId":"vals-biomysterybench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":69.259,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.10526315789474,"participants":20,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 696.818. Standardfehler: 1.335. Kosten je Aufgabe in USD: 2.487742. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 696.818. Standard error: 1.335. Cost per task in USD: 2.487742. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: Terminus 2. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: Terminus 2. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":75.80645161290323,"measurements":[{"observationId":"vals-code-migration:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":44.816,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.80645161290323,"participants":63,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 4141.411. Standardfehler: 4.215. Kosten je Aufgabe in USD: 36.55237. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 4141.411. Standard error: 4.215. Cost per task in USD: 36.55237. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":60.60606060606061,"measurements":[{"observationId":"vals-cyberbench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":69.464,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.60606060606061,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 930.024. Standardfehler: 5.666. Kosten je Aufgabe in USD: 8.539306. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 930.024. Standard error: 5.666. Cost per task in USD: 8.539306. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":72.88135593220339,"measurements":[{"observationId":"vals-emb:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":66.991,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.88135593220339,"participants":60,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2097.023. Standardfehler: 3.045. Kosten je Aufgabe in USD: 6.481284. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2097.023. Standard error: 3.045. Cost per task in USD: 6.481284. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":58.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":52.251,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 932.724. Standardfehler: 0.355. Kosten je Aufgabe in USD: 2.719136. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 932.724. Standard error: 0.355. Cost per task in USD: 2.719136. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-ioi:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":57.722,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":34,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2808.481. Standardfehler: 1.987. Kosten je Aufgabe in USD: 12.707908. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2808.481. Standard error: 1.987. Cost per task in USD: 12.707908. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":88.52459016393442,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":12.5,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.52459016393442,"participants":62,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2525.239. Standardfehler: 2.663. Kosten je Aufgabe in USD: 11.130983. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2525.239. Standard error: 2.663. Cost per task in USD: 11.130983. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.389,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.23076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 26.019. Standardfehler: 0.456. Kosten je Aufgabe in USD: 0.010658. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 26.019. Standard error: 0.456. Cost per task in USD: 0.010658. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":81.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":47.115,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1274.337. Standardfehler: 3.469. Kosten je Aufgabe in USD: 4.920812. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1274.337. Standard error: 3.469. Cost per task in USD: 4.920812. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":76.11940298507463,"measurements":[{"observationId":"vals-medcode:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.553,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.11940298507463,"participants":68,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Latenz in Sekunden: 213.135. Standardfehler: 2.171. Kosten je Aufgabe in USD: 0.105497. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Latency in seconds: 213.135. Standard error: 2.171. Cost per task in USD: 0.105497. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":92.6470588235294,"measurements":[{"observationId":"vals-medscribe:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":89.377,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":92.6470588235294,"participants":69,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Latenz in Sekunden: 153.053. Standardfehler: 1.886. Kosten je Aufgabe in USD: 0.074505. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Latency in seconds: 153.053. Standard error: 1.886. Cost per task in USD: 0.074505. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":35,"measurements":[{"observationId":"vals-mysterymechanism:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":25.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35,"participants":21,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1052.564. Standardfehler: 2.921. Kosten je Aufgabe in USD: 1.841835. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1052.564. Standard error: 2.921. Cost per task in USD: 1.841835. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 13078.924. Standardfehler: 0.5. Kosten je Aufgabe in USD: 46.490946. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 13078.924. Standard error: 0.5. Cost per task in USD: 46.490946. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":32.05128205128205,"measurements":[{"observationId":"vals-proof-bench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":26,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.05128205128205,"participants":40,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 767.177. Standardfehler: 4.408. Kosten je Aufgabe in USD: 0.787951. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 767.177. Standard error: 4.408. Cost per task in USD: 0.787951. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":25.862068965517242,"measurements":[{"observationId":"vals-sage:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":40.79,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.862068965517242,"participants":59,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 290.539. Standardfehler: 3.348. Kosten je Aufgabe in USD: 0.324586. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 290.539. Standard error: 3.348. Cost per task in USD: 0.324586. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":63.1578947368421,"measurements":[{"observationId":"vals-snap:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":65.629,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.1578947368421,"participants":39,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1016.967. Standardfehler: 1.235. Kosten je Aufgabe in USD: 2.246775. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1016.967. Standard error: 1.235. Cost per task in USD: 2.246775. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":39.583333333333336,"measurements":[{"observationId":"vals-sre-bench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":3.817,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.583333333333336,"participants":25,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 6042.744. Standardfehler: 1.186. Kosten je Aufgabe in USD: 64.767258. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 6042.744. Standard error: 1.186. Cost per task in USD: 64.767258. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"vals-swebench-multimodal:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":42.917,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.76923076923077,"participants":14,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 1261.388. Standardfehler: 2.262. Kosten je Aufgabe in USD: 4.662136. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 1261.388. Standard error: 2.262. Cost per task in USD: 4.662136. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":83.63636363636364,"measurements":[{"observationId":"vals-tax-agent-bench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":34.197,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.63636363636364,"participants":56,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 996.469. Standardfehler: 3.423. Kosten je Aufgabe in USD: 1.796539. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 996.469. Standard error: 3.423. Cost per task in USD: 1.796539. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":87.5,"measurements":[{"observationId":"vals-vibe-code-bench:overall:grok/grok-4.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":86.175,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.5,"participants":73,"methodology":{"de":"Kennung des Quellmodells: grok/grok-4.7. Name des Quellmodells: Grok 4.7. Anbieter laut Quelle: SpaceXAI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95. Latenz in Sekunden: 2147.065. Standardfehler: 2.181. Kosten je Aufgabe in USD: 15.82706. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: grok/grok-4.7. Source model name: Grok 4.7. Provider reported by source: SpaceXAI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95. Latency in seconds: 2147.065. Standard error: 2.181. Cost per task in USD: 15.82706. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 0.7. Top P: 0.95","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 0.7. Top P: 0.95"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":58.087256805452846,"coveredFamilies":41,"totalFamilies":71,"sourceCount":4,"coverage":0.5617063492063492,"lowerBound":32.62798095560258,"upperBound":76.45734603496766,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":55.960585253227556,"coveredFamilies":9,"totalFamilies":15,"sourceCount":3,"coverage":0.5888888888888891,"lowerBound":32.95456687134513,"upperBound":74.06567798245621,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":56.91056910569105,"coveredFamilies":5,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":32.52032520325203,"upperBound":75.37746806039489,"provisional":true,"configurationsDiffer":true},"math":{"score":29.496891996891996,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":17.698135198135198,"upperBound":57.69813519813519,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":72.5609756097561,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":48.3739837398374,"upperBound":81.70731707317074,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":35.89965768078734,"upperBound":94.23299101412067,"provisional":true,"configurationsDiffer":true},"general":{"score":32.64437134502924,"coveredFamilies":4,"totalFamilies":5,"sourceCount":2,"coverage":0.8,"lowerBound":26.115497076023395,"upperBound":46.11549707602339,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":30.693007557414333,"upperBound":87.83586470027147,"provisional":true,"configurationsDiffer":true},"legal":{"score":79.80734202045677,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":59.855506515342576,"upperBound":84.85550651534257,"provisional":true,"configurationsDiffer":true},"health":{"score":84.38323090430202,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":56.25548726953468,"upperBound":89.58882060286803,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":60.60606060606061,"upperBound":60.60606060606061,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":11.30952380952381,"upperBound":73.80952380952381,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":31.578947368421048,"upperBound":81.57894736842104,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":12.931034482758621,"upperBound":62.93103448275862,"provisional":true,"configurationsDiffer":true}}},{"modelId":"grok-build-0.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"hemmingway-1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"hermes-4-405b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"hermes-4-70b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"holo4-27b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"holo4-35b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"holotron4-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"humain-m3","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"hunyuan-a13b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"hy3","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":56.09756097560975,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":946.66,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":55.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":79,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 79,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 79.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":47.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":18.409128371172397,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 18,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 18.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":50.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":11.200000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 11,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 11.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1066.59,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":34.146341463414636,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-18.466666666666665,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.146341463414636,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":33.4569045412419,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 33,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 33.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":40.853658536585364,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b23e6c69-96e5-44c9-8f58-4b42e0c399d5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Name des Quellmodells: Hy3. Anbieter laut Quelle: Tencent. Modellstand: Hy3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b23e6c69-96e5-44c9-8f58-4b42e0c399d5/hy3/hy3. Source model name: Hy3. Provider reported by source: Tencent. Model snapshot: Hy3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Hy3 (Release hy3, Zeile hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Hy3 (release hy3, row hy3, UUID b23e6c69-96e5-44c9-8f58-4b42e0c399d5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":3.143977103036337,"upperBound":96.18819479011115,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":1.3617886178861793,"upperBound":98.02845528455285,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":18.46689895470383,"upperBound":82.75261324041813,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":11.382113821138212,"upperBound":78.04878048780489,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":12.804878048780488,"upperBound":87.8048780487805,"provisional":true,"configurationsDiffer":true}}},{"modelId":"ibm-granite-4-h-small","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-homura-2b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-homura-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-nailong-2b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-nailong-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-translate-2b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-translate-35b-a3b-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"index-translate-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"inkling","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":26.31578947368421,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":23.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.31578947368421,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 23,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 23.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":47.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":832.43,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":77.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 77,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 77.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":4.984889684475201,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 5,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 5.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":15.8388333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 15,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 15.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":57.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":12.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 12,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 12.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":51.21951219512195,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":1076.19,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.21951219512195,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":2,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":61.8421052631579,"measurements":[{"observationId":"vals-gpqa:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":87.122,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.8421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 308.595. Standardfehler: 1.784. Kosten je Aufgabe in USD: 0.16589. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 308.595. Standard error: 1.784. Cost per task in USD: 0.16589. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":31.881371640407803,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 31,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 31.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":30.45212765957447,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":73.4682080924856,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.5,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 73,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 73.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":78.497,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.404255319148938,"participants":48,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 123.645. Standardfehler: 0.988. Kosten je Aufgabe in USD: 0.031466. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 123.645. Standard error: 0.988. Cost per task in USD: 0.031466. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":55.932203389830505,"measurements":[{"observationId":"vals-swebench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":77.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.932203389830505,"participants":60,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 394.193. Standardfehler: 1.866. Kosten je Aufgabe in USD: 0.585966. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 394.193. Standard error: 1.866. Cost per task in USD: 0.585966. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":33.84264054995763,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":47.566,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":57,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 771.551. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.630691. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 771.551. Standard error: 0.375. Cost per task in USD: 0.630691. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":0.505,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.756756756756757,"participants":38,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1378.754. Standardfehler: 0.505. Kosten je Aufgabe in USD: 2.495998. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1378.754. Standard error: 0.505. Cost per task in USD: 2.495998. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":4.3478260869565215,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:0de09623-2b1a-4c8d-86ef-7f5245d4e24b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.3478260869565215,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/0de09623-2b1a-4c8d-86ef-7f5245d4e24b/inkling/inkling. Source model name: Inkling. Provider reported by source: Thinking Machines. Model snapshot: Inkling (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling (Xhigh) (Release inkling, Zeile inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Inkling (Xhigh) (release inkling, row inkling, UUID 0de09623-2b1a-4c8d-86ef-7f5245d4e24b, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":19.35483870967742,"measurements":[{"observationId":"vals-code-migration:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":11.79,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.35483870967742,"participants":63,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1773.502. Standardfehler: 3.396. Kosten je Aufgabe in USD: 3.006967. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1773.502. Standard error: 3.396. Cost per task in USD: 3.006967. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":94.5945945945946,"measurements":[{"observationId":"vals-corp-fin-v2:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":68.57,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.5945945945946,"participants":75,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 29.199. Standardfehler: 0.915. Kosten je Aufgabe in USD: 0.084258. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 29.199. Standard error: 0.915. Cost per task in USD: 0.084258. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-cyberbench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":66.786,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":34,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 811.106. Standardfehler: 5.953. Kosten je Aufgabe in USD: 1.463894. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 811.106. Standard error: 5.953. Cost per task in USD: 1.463894. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":20.338983050847457,"measurements":[{"observationId":"vals-emb:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":38.362,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.338983050847457,"participants":60,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1581.835. Standardfehler: 3.273. Kosten je Aufgabe in USD: 1.671783. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1581.835. Standard error: 3.273. Cost per task in USD: 1.671783. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":35,"measurements":[{"observationId":"vals-finance-agent-v2:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":46.597,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35,"participants":61,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1082.125. Standardfehler: 0.925. Kosten je Aufgabe in USD: 0.996264. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1082.125. Standard error: 0.925. Cost per task in USD: 0.996264. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":6.0606060606060606,"measurements":[{"observationId":"vals-ioi:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":14.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.0606060606060606,"participants":34,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1812.242. Standardfehler: 5.403. Kosten je Aufgabe in USD: 1.70838. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1812.242. Standard error: 5.403. Cost per task in USD: 1.70838. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":70.12987012987013,"measurements":[{"observationId":"vals-lcb:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.517,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.12987012987013,"participants":78,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 257.712. Standardfehler: 1.003. Kosten je Aufgabe in USD: 0.153251. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 257.712. Standard error: 1.003. Cost per task in USD: 0.153251. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":43.442622950819676,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.083,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.442622950819676,"participants":62,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1103.692. Standardfehler: 0.833. Kosten je Aufgabe in USD: 1.252023. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1103.692. Standard error: 0.833. Cost per task in USD: 1.252023. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"vals-legal-bench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.949,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":79,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 23.142. Standardfehler: 0.496. Kosten je Aufgabe in USD: 0.013327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 23.142. Standard error: 0.496. Cost per task in USD: 0.013327. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":40,"measurements":[{"observationId":"vals-legal-research:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":28.365,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40,"participants":61,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 869.377. Standardfehler: 3.133. Kosten je Aufgabe in USD: 0.494044. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 869.377. Standard error: 3.133. Cost per task in USD: 0.494044. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":40.298507462686565,"measurements":[{"observationId":"vals-medcode:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":41.19,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.298507462686565,"participants":68,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 167.123. Standardfehler: 2.23. Kosten je Aufgabe in USD: 0.127025. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 167.123. Standard error: 2.23. Cost per task in USD: 0.127025. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":70.58823529411765,"measurements":[{"observationId":"vals-medscribe:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":85.405,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.58823529411765,"participants":69,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 244.954. Standardfehler: 1.844. Kosten je Aufgabe in USD: 0.165561. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 244.954. Standard error: 1.844. Cost per task in USD: 0.165561. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":62.33766233766234,"measurements":[{"observationId":"vals-mmlu-pro:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":86.295,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.33766233766234,"participants":78,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 73.264. Standardfehler: 0.34. Kosten je Aufgabe in USD: 0.039629. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 73.264. Standard error: 0.34. Cost per task in USD: 0.039629. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":29.807692307692307,"measurements":[{"observationId":"vals-mortgage-tax:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":63.99,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.807692307692307,"participants":53,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 69.108. Standardfehler: 0.944. Kosten je Aufgabe in USD: 0.056492. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 69.108. Standard error: 0.944. Cost per task in USD: 0.056492. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 2734.146. Standardfehler: 0. Kosten je Aufgabe in USD: 4.143489. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 2734.146. Standard error: 0. Cost per task in USD: 4.143489. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":1.2820512820512822,"measurements":[{"observationId":"vals-proof-bench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.2820512820512822,"participants":40,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 436.43. Standardfehler: 0. Kosten je Aufgabe in USD: 0.201396. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 436.43. Standard error: 0. Cost per task in USD: 0.201396. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":17.24137931034483,"measurements":[{"observationId":"vals-sage:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":36.555,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.24137931034483,"participants":59,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 198.023. Standardfehler: 3.216. Kosten je Aufgabe in USD: 0.163151. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 198.023. Standard error: 3.216. Cost per task in USD: 0.163151. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":3.4482758620689653,"measurements":[{"observationId":"vals-skillsbench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":26.091,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.4482758620689653,"participants":30,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 938.657. Standardfehler: 3.863. Kosten je Aufgabe in USD: 0.826622. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 938.657. Standard error: 3.863. Cost per task in USD: 0.826622. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":26.31578947368421,"measurements":[{"observationId":"vals-snap:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":58.457,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.31578947368421,"participants":39,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 810.832. Standardfehler: 1.282. Kosten je Aufgabe in USD: 0.367636. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 810.832. Standard error: 1.282. Cost per task in USD: 0.367636. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":4.166666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.166666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1718.686. Standardfehler: 0. Kosten je Aufgabe in USD: 2.225036. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1718.686. Standard error: 0. Cost per task in USD: 2.225036. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":30.90909090909091,"measurements":[{"observationId":"vals-tax-agent-bench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":15.291,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.90909090909091,"participants":56,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 810.557. Standardfehler: 2.423. Kosten je Aufgabe in USD: 0.353757. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 810.557. Standard error: 2.423. Cost per task in USD: 0.353757. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":84.86842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.306,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.86842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 197.78. Standardfehler: 0.839. Kosten je Aufgabe in USD: 0.103224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 197.78. Standard error: 0.839. Cost per task in USD: 0.103224. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":4.761904761904762,"measurements":[{"observationId":"vals-vcb-1-100:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":7.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.761904761904762,"participants":22,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 3001.539. Standardfehler: 1.839. Kosten je Aufgabe in USD: 4.997334. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 3001.539. Standard error: 1.839. Cost per task in USD: 4.997334. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":22.22222222222222,"measurements":[{"observationId":"vals-vibe-code-bench:overall:thinkingmachines/inkling","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":19.214,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.22222222222222,"participants":73,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling. Name des Quellmodells: Inkling. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1105.492. Standardfehler: 3.659. Kosten je Aufgabe in USD: 1.605949. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling. Source model name: Inkling. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000. Latency in seconds: 1105.492. Standard error: 3.659. Cost per task in USD: 1.605949. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]}],"overall":{"score":39.5973905360264,"coveredFamilies":38,"totalFamilies":71,"sourceCount":2,"coverage":0.5017573696145126,"lowerBound":19.8682825189552,"upperBound":69.69254555750395,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":23.248466377283712,"coveredFamilies":10,"totalFamilies":15,"sourceCount":2,"coverage":0.5888888888888891,"lowerBound":13.690763533289303,"upperBound":54.801874644400385,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":22.797084173849253,"upperBound":79.9399413167064,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.25641025641025644,"upperBound":80.25641025641026,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":61.04687994931897,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":40.697919966212645,"upperBound":74.03125329954598,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":13.455948534076995,"upperBound":71.78928186741032,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.8333333333333335,"upperBound":80.83333333333333,"provisional":true,"configurationsDiffer":true},"finance":{"score":49.25313031914281,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":42.21696884497955,"upperBound":56.50268313069384,"provisional":true,"configurationsDiffer":true},"legal":{"score":43.198823034888605,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":32.39911727616646,"upperBound":57.39911727616646,"provisional":true,"configurationsDiffer":true},"health":{"score":55.443371378402105,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":36.9622475856014,"upperBound":70.29558091893475,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":45.45454545454545,"upperBound":45.45454545454545,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":7.613031914893617,"upperBound":82.61303191489361,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":13.157894736842104,"upperBound":63.1578947368421,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":8.620689655172415,"upperBound":58.62068965517241,"provisional":true,"configurationsDiffer":true}}},{"modelId":"inkling-small","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":31.57894736842105,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":27.500000000000004,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.57894736842105,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 27,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 27.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":52.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":911.75,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":75.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 75,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 75.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":4.814227515250378,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 4,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 4.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":50.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":11.200000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 11,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 11.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":54.8780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1123.9,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.8780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":47.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-8.933333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":46.05263157894737,"measurements":[{"observationId":"vals-gpqa:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":83.586,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 259.758. Standardfehler: 2.036. Kosten je Aufgabe in USD: 0.020022. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 259.758. Standard error: 2.036. Cost per task in USD: 0.020022. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":33.3178869323448,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 33,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 33.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":39.285714285714285,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":74.0462427745665,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.285714285714285,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 74,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 74.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":74.57627118644068,"measurements":[{"observationId":"vals-swebench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":82.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.57627118644068,"participants":60,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 245.037. Standardfehler: 1.712. Kosten je Aufgabe in USD: 0.13413. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 245.037. Standard error: 1.712. Cost per task in USD: 0.13413. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":36.69013089744798,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":55.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.92857142857143,"participants":57,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 603.689. Standardfehler: 2.339. Kosten je Aufgabe in USD: 0.127848. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 603.689. Standard error: 2.339. Cost per task in USD: 0.127848. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3513513513513513,"participants":38,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 705.115. Standardfehler: 0. Kosten je Aufgabe in USD: 0.321679. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 705.115. Standard error: 0. Cost per task in USD: 0.321679. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":4.3478260869565215,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:7261504e-503c-4a66-a9d3-a3272cdf9ad6","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.3478260869565215,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinking Machines. Modellstand: Inkling Small. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/7261504e-503c-4a66-a9d3-a3272cdf9ad6/inkling-small/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinking Machines. Model snapshot: Inkling Small. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Inkling Small (Release inkling-small, Zeile inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Inkling Small (release inkling-small, row inkling-small, UUID 7261504e-503c-4a66-a9d3-a3272cdf9ad6, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":25.806451612903224,"measurements":[{"observationId":"vals-code-migration:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":13.675,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.806451612903224,"participants":63,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 1232.702. Standardfehler: 3.81. Kosten je Aufgabe in USD: 0.596991. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 1232.702. Standard error: 3.81. Cost per task in USD: 0.596991. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":95.94594594594595,"measurements":[{"observationId":"vals-corp-fin-v2:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":69.619,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95.94594594594595,"participants":75,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 45.316. Standardfehler: 0.906. Kosten je Aufgabe in USD: 0.025455. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 45.316. Standard error: 0.906. Cost per task in USD: 0.025455. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-cyberbench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":64.107,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":34,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 1175.122. Standardfehler: 6.17. Kosten je Aufgabe in USD: 0.251693. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 1175.122. Standard error: 6.17. Cost per task in USD: 0.251693. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":18.64406779661017,"measurements":[{"observationId":"vals-emb:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":31.762,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.64406779661017,"participants":60,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 1173.625. Standardfehler: 2.956. Kosten je Aufgabe in USD: 0.249301. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 1173.625. Standard error: 2.956. Cost per task in USD: 0.249301. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":21.666666666666668,"measurements":[{"observationId":"vals-finance-agent-v2:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":41.257,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 994.255. Standardfehler: 0.832. Kosten je Aufgabe in USD: 0.230271. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 994.255. Standard error: 0.832. Cost per task in USD: 0.230271. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":3.0303030303030303,"measurements":[{"observationId":"vals-ioi:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":9.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.0303030303030303,"participants":34,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 678.056. Standardfehler: 4.475. Kosten je Aufgabe in USD: 0.337871. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 678.056. Standard error: 4.475. Cost per task in USD: 0.337871. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":75.32467532467533,"measurements":[{"observationId":"vals-lcb:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.931,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.32467532467533,"participants":78,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 302.324. Standardfehler: 0.992. Kosten je Aufgabe in USD: 0.020896. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 302.324. Standard error: 0.992. Cost per task in USD: 0.020896. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":37.704918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.704918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 1870.966. Standardfehler: 0.833. Kosten je Aufgabe in USD: 0.174542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 1870.966. Standard error: 0.833. Cost per task in USD: 0.174542. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":47.43589743589744,"measurements":[{"observationId":"vals-legal-bench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.986,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.43589743589744,"participants":79,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 17.655. Standardfehler: 0.459. Kosten je Aufgabe in USD: 0.001332. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 17.655. Standard error: 0.459. Cost per task in USD: 0.001332. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":35.833333333333336,"measurements":[{"observationId":"vals-legal-research:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":25.481,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.833333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 774.823. Standardfehler: 3.029. Kosten je Aufgabe in USD: 0.141548. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 774.823. Standard error: 3.029. Cost per task in USD: 0.141548. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":20.895522388059703,"measurements":[{"observationId":"vals-medcode:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":37.893,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.895522388059703,"participants":68,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 191.049. Standardfehler: 2.206. Kosten je Aufgabe in USD: 0.016656. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 191.049. Standard error: 2.206. Cost per task in USD: 0.016656. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":61.76470588235294,"measurements":[{"observationId":"vals-medscribe:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":84.112,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.76470588235294,"participants":69,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 229.993. Standardfehler: 1.87. Kosten je Aufgabe in USD: 0.019018. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 229.993. Standard error: 1.87. Cost per task in USD: 0.019018. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":46.753246753246756,"measurements":[{"observationId":"vals-mmlu-pro:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.566,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.753246753246756,"participants":78,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 71.685. Standardfehler: 0.346. Kosten je Aufgabe in USD: 0.005706. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 71.685. Standard error: 0.346. Cost per task in USD: 0.005706. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":25,"measurements":[{"observationId":"vals-mortgage-tax:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":62.282,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":53,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 112.214. Standardfehler: 0.952. Kosten je Aufgabe in USD: 0.011526. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 112.214. Standard error: 0.952. Cost per task in USD: 0.011526. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 1710.815. Standardfehler: 0.5. Kosten je Aufgabe in USD: 0.479317. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 1710.815. Standard error: 0.5. Cost per task in USD: 0.479317. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"vals-proof-bench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":40,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 507.301. Standardfehler: 2.387. Kosten je Aufgabe in USD: 0.075386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 507.301. Standard error: 2.387. Cost per task in USD: 0.075386. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":8.620689655172415,"measurements":[{"observationId":"vals-sage:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":32.744,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.620689655172415,"participants":59,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 206.475. Standardfehler: 3.258. Kosten je Aufgabe in USD: 0.022636. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000. Latency in seconds: 206.475. Standard error: 3.258. Cost per task in USD: 0.022636. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 30000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":10.344827586206897,"measurements":[{"observationId":"vals-skillsbench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":33.582,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.344827586206897,"participants":30,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 700.919. Standardfehler: 3.66. Kosten je Aufgabe in USD: 0.148059. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 700.919. Standard error: 3.66. Cost per task in USD: 0.148059. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":31.57894736842105,"measurements":[{"observationId":"vals-snap:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":59.405,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.57894736842105,"participants":39,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 521.336. Standardfehler: 1.277. Kosten je Aufgabe in USD: 0.095937. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 521.336. Standard error: 1.277. Cost per task in USD: 0.095937. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":32.72727272727273,"measurements":[{"observationId":"vals-tax-agent-bench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":15.389,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.72727272727273,"participants":56,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 385.252. Standardfehler: 2.365. Kosten je Aufgabe in USD: 0.091767. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 385.252. Standard error: 2.365. Cost per task in USD: 0.091767. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":86.84210526315789,"measurements":[{"observationId":"vals-tax-eval-v2:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.511,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.84210526315789,"participants":77,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 241.592. Standardfehler: 0.837. Kosten je Aufgabe in USD: 0.017501. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 241.592. Standard error: 0.837. Cost per task in USD: 0.017501. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":20.833333333333332,"measurements":[{"observationId":"vals-vibe-code-bench:overall:thinkingmachines/inkling-small","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":19.052,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.833333333333332,"participants":73,"methodology":{"de":"Kennung des Quellmodells: thinkingmachines/inkling-small. Name des Quellmodells: Inkling Small. Anbieter laut Quelle: Thinkingmachines. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000. Latenz in Sekunden: 2949.469. Standardfehler: 3.403. Kosten je Aufgabe in USD: 0.432264. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: thinkingmachines/inkling-small. Source model name: Inkling Small. Provider reported by source: Thinkingmachines. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000. Latency in seconds: 2949.469. Standard error: 3.403. Cost per task in USD: 0.432264. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: 0.99. Temperatur: 1. Top P: 1. Maximale Ausgabetoken: 262000","en":"Harness: OpenHands. Reasoning effort: 0.99. Temperature: 1. Top P: 1. Maximum output tokens: 262000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"0.99","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":35,"totalFamilies":71,"sourceCount":2,"coverage":0.46901927437641733,"lowerBound":17.410738951738576,"upperBound":70.50881151409685,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":2,"coverage":0.4555555555555557,"lowerBound":17.01573351011822,"upperBound":71.46017795456265,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":19.49156427654502,"upperBound":76.63442141940216,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.5384615384615385,"upperBound":81.53846153846153,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":47.15711118150143,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":31.438074121000952,"upperBound":64.77140745433428,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":14.977387750284038,"upperBound":73.31072108361737,"provisional":true,"configurationsDiffer":true},"finance":{"score":46.8043430666089,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":40.11800834280763,"upperBound":54.40372262852192,"provisional":true,"configurationsDiffer":true},"legal":{"score":40.32471626733922,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":30.243537200504417,"upperBound":55.24353720050442,"provisional":true,"configurationsDiffer":true},"health":{"score":41.33011413520632,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":27.55340942347088,"upperBound":60.88674275680422,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":36.36363636363637,"upperBound":36.36363636363637,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":4.910714285714286,"upperBound":92.41071428571429,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":15.789473684210524,"upperBound":65.78947368421052,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":4.310344827586207,"upperBound":54.310344827586206,"provisional":true,"configurationsDiffer":true}}},{"modelId":"iquest-q1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"jamba-large-1-7","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"jamba-mini-1-7","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"jamba-reasoning-3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"jamba2-3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"jamba2-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-0.9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-3.7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-375b-a23b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"k2-horizon-mova-36b-a4b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"kat-coder-air-v2.5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"kat-coder-pro-v2.5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"kimi-k2-0711","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":19.736842105263158,"measurements":[{"observationId":"vals-gpqa:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":71.464,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.736842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 41.434. Standardfehler: 2.266. Kosten je Aufgabe in USD: 0.004481. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 41.434. Standard error: 2.266. Cost per task in USD: 0.004481. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":19.047619047619047,"measurements":[{"observationId":"vals-terminal-bench-2:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":25.843,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.047619047619047,"participants":43,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.3. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 702.2. Standardfehler: 4.667. Kosten je Aufgabe in USD: 1.107135. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.3. Maximum output tokens: 16384. Latency in seconds: 702.2. Standard error: 4.667. Cost per task in USD: 1.107135. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 16384","en":"Temperature: 0.3. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 16384","en":"Temperature: 0.3. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":10.81081081081081,"measurements":[{"observationId":"vals-corp-fin-v2:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":50.388,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.81081081081081,"participants":75,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.3. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 40.894. Standardfehler: 0.985. Kosten je Aufgabe in USD: 0.059336. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.3. Maximum output tokens: 16384. Latency in seconds: 40.894. Standard error: 0.985. Cost per task in USD: 0.059336. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 16384","en":"Temperature: 0.3. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Maximale Ausgabetoken: 16384","en":"Temperature: 0.3. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":24.675324675324674,"measurements":[{"observationId":"vals-lcb:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":70.449,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.675324675324674,"participants":78,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 66.651. Standardfehler: 1.159. Kosten je Aufgabe in USD: 0.006889. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 66.651. Standard error: 1.159. Cost per task in USD: 0.006889. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":38.46153846153846,"measurements":[{"observationId":"vals-legal-bench:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":81.454,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.46153846153846,"participants":79,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 1.172. Standardfehler: 0.444. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 1.172. Standard error: 0.444. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"vals-math500:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.27272727272727,"participants":23,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 37.346. Standardfehler: 1.045. Kosten je Aufgabe in USD: 0.002985. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 37.346. Standard error: 1.045. Cost per task in USD: 0.002985. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"vals-medqa:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":83.975,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.76923076923077,"participants":40,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 10.984. Standardfehler: 0.335. Kosten je Aufgabe in USD: 0.001028. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 10.984. Standard error: 0.335. Cost per task in USD: 0.001028. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":62.06896551724138,"measurements":[{"observationId":"vals-mgsm:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":90.946,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.06896551724138,"participants":30,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 11.83. Standardfehler: 1.766. Kosten je Aufgabe in USD: 0.001001. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 11.83. Standard error: 1.766. Cost per task in USD: 0.001001. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":19.48051948051948,"measurements":[{"observationId":"vals-mmlu-pro:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":79.394,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.48051948051948,"participants":78,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 20.338. Standardfehler: 0.394. Kosten je Aufgabe in USD: 0.002836. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 20.338. Standard error: 0.394. Cost per task in USD: 0.002836. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":23.68421052631579,"measurements":[{"observationId":"vals-tax-eval-v2:overall:together/moonshotai/Kimi-K2-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":70.196,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.68421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: together/moonshotai/Kimi-K2-Instruct. Name des Quellmodells: Kimi K2 0711. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 12.834. Standardfehler: 0.895. Kosten je Aufgabe in USD: 0.001543. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/moonshotai/Kimi-K2-Instruct. Source model name: Kimi K2 0711. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32768. Latency in seconds: 12.834. Standard error: 0.895. Cost per task in USD: 0.001543. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32768","en":"Temperature: 0.6. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.1251133786848073,"lowerBound":4.459138314790377,"upperBound":91.94780044630964,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":1,"coverage":0.0777777777777778,"lowerBound":1.8566618566618573,"upperBound":94.07888407888407,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.4097744360902256,"upperBound":94.26691729323309,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":27.86833855799373,"upperBound":87.86833855799372,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":6.493506493506493,"upperBound":73.16017316017316,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":4.927860191018086,"upperBound":76.35643161958951,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":9.615384615384615,"upperBound":84.61538461538461,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":10.256410256410255,"upperBound":76.92307692307693,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2-0905","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:66445f84-b2e3-4202-afdc-92ba0f0e5f36","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":41.7006802721088,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.333333333333336,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2 0905 (Release kimi-k2-0905, Zeile kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/66445f84-b2e3-4202-afdc-92ba0f0e5f36/kimi-k2-0905/kimi-k2-0905. Name des Quellmodells: Kimi K2 0905. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2 0905. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 41,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2 0905 (release kimi-k2-0905, row kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/66445f84-b2e3-4202-afdc-92ba0f0e5f36/kimi-k2-0905/kimi-k2-0905. Source model name: Kimi K2 0905. Provider reported by source: Kimi. Model snapshot: Kimi K2 0905. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 41.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2 0905 (Release kimi-k2-0905, Zeile kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Kimi K2 0905 (release kimi-k2-0905, row kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2 0905 (Release kimi-k2-0905, Zeile kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Kimi K2 0905 (release kimi-k2-0905, row kimi-k2-0905, UUID 66445f84-b2e3-4202-afdc-92ba0f0e5f36, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.4761904761904763,"upperBound":99.04761904761907,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.666666666666668,"upperBound":86.66666666666667,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2-thinking","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":60.869565217391305,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":68.0952380952381,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.869565217391305,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2 Thinking (Release kimi-k2-thinking, Zeile kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc/kimi-k2-thinking/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2 Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 68,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2 Thinking (release kimi-k2-thinking, row kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc/kimi-k2-thinking/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Kimi. Model snapshot: Kimi K2 Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 68.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2 Thinking (Release kimi-k2-thinking, Zeile kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2 Thinking (release kimi-k2-thinking, row kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2 Thinking (Release kimi-k2-thinking, Zeile kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2 Thinking (release kimi-k2-thinking, row kimi-k2-thinking, UUID bddebfd3-0a8d-47f5-b722-bc4c2ca5a5dc, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":31.57894736842105,"measurements":[{"observationId":"vals-gpqa:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":78.536,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.57894736842105,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 585.555. Standardfehler: 2.183. Kosten je Aufgabe in USD: 0.023072. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 585.555. Standard error: 2.183. Cost per task in USD: 0.023072. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":8.474576271186441,"measurements":[{"observationId":"vals-swebench:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":60.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.474576271186441,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32000. Latenz in Sekunden: 1632.209. Standardfehler: 2.191. Kosten je Aufgabe in USD: 0.366026. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32000. Latency in seconds: 1632.209. Standard error: 2.191. Cost per task in USD: 0.366026. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":35.714285714285715,"measurements":[{"observationId":"vals-terminal-bench-2:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":37.079,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.714285714285715,"participants":43,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 986.553. Standardfehler: 5.149. Kosten je Aufgabe in USD: 0.115876. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Maximum output tokens: 128000. Latency in seconds: 986.553. Standard error: 5.149. Cost per task in USD: 0.115876. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":37.83783783783784,"measurements":[{"observationId":"vals-corp-fin-v2:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":60.567,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.83783783783784,"participants":75,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.6. Maximale Ausgabetoken: 32000. Latenz in Sekunden: 93.297. Standardfehler: 0.962. Kosten je Aufgabe in USD: 0.038163. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.6. Maximum output tokens: 32000. Latency in seconds: 93.297. Standard error: 0.962. Cost per task in USD: 0.038163. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":15.584415584415584,"measurements":[{"observationId":"vals-lcb:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":63.145,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.584415584415584,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 1150.624. Standardfehler: 1.157. Kosten je Aufgabe in USD: 0.033631. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 1150.624. Standard error: 1.157. Cost per task in USD: 0.033631. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"vals-legal-bench:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":80.201,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":79,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 43.991. Standardfehler: 0.459. Kosten je Aufgabe in USD: 0.002182. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 43.991. Standard error: 0.459. Cost per task in USD: 0.002182. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"vals-medqa:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":92.592,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":40,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 225.305. Standardfehler: 0.239. Kosten je Aufgabe in USD: 0.006153. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Maximum output tokens: 32768. Latency in seconds: 225.305. Standard error: 0.239. Cost per task in USD: 0.006153. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32768","en":"Temperature: 1. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":51.724137931034484,"measurements":[{"observationId":"vals-mgsm:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":90.146,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.724137931034484,"participants":30,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Maximale Ausgabetoken: 32000. Latenz in Sekunden: 48.41. Standardfehler: 1.83. Kosten je Aufgabe in USD: 0.000891. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Maximum output tokens: 32000. Latency in seconds: 48.41. Standard error: 1.83. Cost per task in USD: 0.000891. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32000","en":"Temperature: 1. Maximum output tokens: 32000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 32000","en":"Temperature: 1. Maximum output tokens: 32000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":28.571428571428573,"measurements":[{"observationId":"vals-mmlu-pro:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":81.068,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.571428571428573,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 94.771. Standardfehler: 0.396. Kosten je Aufgabe in USD: 0.002067. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 94.771. Standard error: 0.396. Cost per task in USD: 0.002067. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-poker-agent","area":"agentic","label":{"de":"Agent Poker Bench","en":"Agent Poker Bench"},"weight":0.005952380952380952,"coverage":1,"score":12.5,"measurements":[{"observationId":"vals-poker-agent:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/poker_agent","benchmarkId":"vals-poker-agent","benchmarkVersion":"1","comparisonCohort":"vals-poker-agent:1:overall","retrievedAt":"2026-10-15","value":1011.6338984569229,"unit":"rating","metric":"TrueSkill rating","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.5,"participants":9,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2025-12-23. Latenz in Sekunden: 71.72413331456877. Standardfehler: 0. Kosten je Aufgabe in USD: 0.02410503610648917. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-poker_agent-2026-10-15-23821a0049c3. Prüfsumme der Quelldaten: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2025-12-23. Latency in seconds: 71.72413331456877. Standard error: 0. Cost per task in USD: 0.02410503610648917. Source ID: vals-ai. Evidence capture ID: vals-poker_agent-2026-10-15-23821a0049c3. Source payload hash: 23821a0049c3ab8467573a74dac6c5037748e301e2abcd91885ec708168335af. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-poker-agent:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":38.1578947368421,"measurements":[{"observationId":"vals-tax-eval-v2:overall:kimi/kimi-k2-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.709,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.1578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2-thinking. Name des Quellmodells: Kimi K2 Thinking. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Maximale Ausgabetoken: 32000. Latenz in Sekunden: 61.408. Standardfehler: 0.878. Kosten je Aufgabe in USD: 0.00913. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2-thinking. Source model name: Kimi K2 Thinking. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.6. Maximum output tokens: 32000. Latency in seconds: 61.408. Standard error: 0.878. Cost per task in USD: 0.00913. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Maximale Ausgabetoken: 32000","en":"Temperature: 0.6. Maximum output tokens: 32000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":2,"coverage":0.13344671201814065,"lowerBound":5.482915378607606,"upperBound":92.13824417679353,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":1,"coverage":0.11111111111111113,"lowerBound":1.7182723114926508,"upperBound":90.60716120038154,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":2.255639097744361,"upperBound":95.11278195488723,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":10.344827586206897,"upperBound":90.34482758620689,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.523809523809524,"upperBound":76.19047619047619,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":1.0416666666666667,"upperBound":92.70833333333333,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":12.173913043478262,"upperBound":92.17391304347827,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":10.856533224954276,"upperBound":82.2851046535257,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":8.333333333333334,"upperBound":83.33333333333333,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":20.51282051282051,"upperBound":87.17948717948718,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2.5","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:a550ffca-f89e-4381-ade6-a85dc6a1fb4c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2968:suffix=Reasoning","retrievedAt":"2026-10-19","value":11.504424778761061,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.384615384615385,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.5 (Reasoning) (Release kimi-k2-5, Zeile kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/a550ffca-f89e-4381-ade6-a85dc6a1fb4c/kimi-k2-5/kimi-k2-5. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.5 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 11,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.5 (Reasoning) (release kimi-k2-5, row kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/a550ffca-f89e-4381-ade6-a85dc6a1fb4c/kimi-k2-5/kimi-k2-5. Source model name: Kimi K2.5. Provider reported by source: Kimi. Model snapshot: Kimi K2.5 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 11.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.5 (Reasoning) (Release kimi-k2-5, Zeile kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.5 (Reasoning) (release kimi-k2-5, row kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.5 (Reasoning) (Release kimi-k2-5, Zeile kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.5 (Reasoning) (release kimi-k2-5, row kimi-k2-5, UUID a550ffca-f89e-4381-ade6-a85dc6a1fb4c, reasoning=true, source reasoning-token configuration budget=2968, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":49.3421052631579,"measurements":[{"observationId":"vals-gpqa:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":84.091,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.3421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 151.843. Standardfehler: 2.135. Kosten je Aufgabe in USD: 0.029962. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 151.843. Standard error: 2.135. Cost per task in USD: 0.029962. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":57.4468085106383,"measurements":[{"observationId":"vals-mmmu:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":84.335,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.4468085106383,"participants":48,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 110.011. Standardfehler: 0.874. Kosten je Aufgabe in USD: 0.019563. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 110.011. Standard error: 0.874. Cost per task in USD: 0.019563. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":26.271186440677965,"measurements":[{"observationId":"vals-swebench:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":70,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.271186440677965,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 339.242. Standardfehler: 2.051. Kosten je Aufgabe in USD: 0.167669. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 339.242. Standard error: 2.051. Cost per task in USD: 0.167669. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":29.315476190476186,"measurements":[{"observationId":"vals-terminal-bench-2:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":40.449,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.80952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 756.405. Standardfehler: 5.232. Kosten je Aufgabe in USD: 0.139997. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 756.405. Standard error: 5.232. Cost per task in USD: 0.139997. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":41.948,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.821428571428571,"participants":57,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1051.079. Standardfehler: 0.749. Kosten je Aufgabe in USD: 0.085844. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 1051.079. Standard error: 0.749. Cost per task in USD: 0.085844. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":16.129032258064516,"measurements":[{"observationId":"vals-code-migration:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":6.962,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.129032258064516,"participants":63,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3820.711. Standardfehler: 1.29. Kosten je Aufgabe in USD: 0.926909. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3820.711. Standard error: 1.29. Cost per task in USD: 0.926909. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":90.54054054054055,"measurements":[{"observationId":"vals-corp-fin-v2:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":68.259,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.54054054054055,"participants":75,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 80.779. Standardfehler: 0.917. Kosten je Aufgabe in USD: 0.052412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 80.779. Standard error: 0.917. Cost per task in USD: 0.052412. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":15.254237288135593,"measurements":[{"observationId":"vals-emb:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":28.469,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.254237288135593,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3796.855. Standardfehler: 2.23. Kosten je Aufgabe in USD: 0.786639. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3796.855. Standard error: 2.23. Cost per task in USD: 0.786639. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":11.666666666666666,"measurements":[{"observationId":"vals-finance-agent-v2:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":35.792,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.666666666666666,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 751.754. Standardfehler: 1.989. Kosten je Aufgabe in USD: 0.294389. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 751.754. Standard error: 1.989. Cost per task in USD: 0.294389. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":57.142857142857146,"measurements":[{"observationId":"vals-lcb:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":83.868,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.142857142857146,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 200.023. Standardfehler: 1.03. Kosten je Aufgabe in USD: 0.038656. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 200.023. Standard error: 1.03. Cost per task in USD: 0.038656. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1508.275. Standardfehler: 0. Kosten je Aufgabe in USD: 0.351359. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1508.275. Standard error: 0. Cost per task in USD: 0.351359. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":23.333333333333332,"measurements":[{"observationId":"vals-legal-research:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":15.865,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1332.514. Standardfehler: 2.539. Kosten je Aufgabe in USD: 0.23035. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1332.514. Standard error: 2.539. Cost per task in USD: 0.23035. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":26.865671641791046,"measurements":[{"observationId":"vals-medcode:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":39.316,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.865671641791046,"participants":68,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 75.452. Standardfehler: 2.119. Kosten je Aufgabe in USD: 0.017275. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 75.452. Standard error: 2.119. Cost per task in USD: 0.017275. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":74.35897435897436,"measurements":[{"observationId":"vals-medqa:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":94.367,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.35897435897436,"participants":40,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 41.628. Standardfehler: 0.21. Kosten je Aufgabe in USD: 0.006992. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 41.628. Standard error: 0.21. Cost per task in USD: 0.006992. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":27.941176470588236,"measurements":[{"observationId":"vals-medscribe:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":76.442,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.941176470588236,"participants":69,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 147.155. Standardfehler: 1.986. Kosten je Aufgabe in USD: 0.02489. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 147.155. Standard error: 1.986. Cost per task in USD: 0.02489. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-mmlu-pro:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.914,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 39.204. Standardfehler: 0.343. Kosten je Aufgabe in USD: 0.00793. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 39.204. Standard error: 0.343. Cost per task in USD: 0.00793. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":53.84615384615385,"measurements":[{"observationId":"vals-mortgage-tax:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":66.534,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.84615384615385,"participants":53,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 79.787. Standardfehler: 0.925. Kosten je Aufgabe in USD: 0.014918. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 79.787. Standard error: 0.925. Cost per task in USD: 0.014918. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":74.13793103448276,"measurements":[{"observationId":"vals-sage:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":49.865,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.13793103448276,"participants":59,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 173.179. Standardfehler: 3.363. Kosten je Aufgabe in USD: 0.028736. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 173.179. Standard error: 3.363. Cost per task in USD: 0.028736. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":63.1578947368421,"measurements":[{"observationId":"vals-tax-eval-v2:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.202,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.1578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 90.464. Standardfehler: 0.846. Kosten je Aufgabe in USD: 0.014221. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 90.464. Standard error: 0.846. Cost per task in USD: 0.014221. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":19.444444444444443,"measurements":[{"observationId":"vals-vibe-code-bench:overall:kimi/kimi-k2.5-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":17.536,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.444444444444443,"participants":73,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.5-thinking. Name des Quellmodells: Kimi K2.5. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2570.381. Standardfehler: 3.258. Kosten je Aufgabe in USD: 0.879292. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.5-thinking. Source model name: Kimi K2.5. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2570.381. Standard error: 3.258. Cost per task in USD: 0.879292. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":20,"totalFamilies":71,"sourceCount":2,"coverage":0.25592403628117916,"lowerBound":11.428141761127025,"upperBound":85.8357381330091,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":1,"coverage":0.25555555555555565,"lowerBound":7.7082501641687005,"upperBound":82.15269460861313,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":3.5244360902255636,"upperBound":96.38157894736842,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":18.181818181818183,"upperBound":84.84848484848486,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":1.282051282051282,"upperBound":92.94871794871794,"provisional":true,"configurationsDiffer":true},"finance":{"score":46.89309861566775,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":33.49507043976268,"upperBound":62.066499011191254,"provisional":true,"configurationsDiffer":true},"legal":{"score":16.99453551912568,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":8.49726775956284,"upperBound":58.49726775956284,"provisional":true,"configurationsDiffer":true},"health":{"score":43.05527415711788,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":43.05527415711788,"upperBound":43.05527415711788,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":7.180851063829788,"upperBound":94.68085106382979,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":37.06896551724138,"upperBound":87.06896551724138,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2.6","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":819.82,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":81,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 81,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 81.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":28.4660766961652,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":61.53846153846154,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 28,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 28.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":13.020759856386269,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 13,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 13.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":13,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 13,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 13.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":1038.29,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":20,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":31.186440677966097,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 31,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 31.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":5.3,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":69.73684210526316,"measurements":[{"observationId":"vals-gpqa:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":89.142,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 544.744. Standardfehler: 1.997. Kosten je Aufgabe in USD: 0.080129. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 544.744. Standard error: 1.997. Cost per task in USD: 0.080129. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":57.31707317073171,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":37.488415199258604,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.31707317073171,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 37,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 37.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"lmarena:webdev:overall:kimi-k2.6","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1509.2367206242538,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":33.333333333333336,"participants":37,"methodology":{"de":"Arena-Bewertung aus 10.192 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Modellstand: kimi-k2.6. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 10,192 published pairwise votes.. Source model ID: lmarena/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Model snapshot: kimi-k2.6. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 10.192 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 10,192 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 10.192 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 10,192 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":30.555555555555557,"measurements":[{"observationId":"lmarena:overall:kimi-k2.6","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1460.759863002005,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":30.555555555555557,"participants":37,"methodology":{"de":"Arena-Bewertung aus 40.030 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Modellstand: kimi-k2.6. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 40,030 published pairwise votes.. Source model ID: lmarena/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Model snapshot: kimi-k2.6. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 40.030 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 40,030 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 40.030 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 40,030 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":36.666666666666664,"measurements":[{"observationId":"lmarena:vision_style_control:overall:kimi-k2.6","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1265.372718816867,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":36.666666666666664,"participants":31,"methodology":{"de":"Arena-Bewertung aus 16.379 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Modellstand: kimi-k2.6. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 16,379 published pairwise votes.. Source model ID: lmarena/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Model snapshot: kimi-k2.6. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 16.379 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 16,379 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 16.379 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 16,379 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":66.90729483282675,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":79.364161849711,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.85714285714286,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 79,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 79.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":86.301,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.95744680851064,"participants":48,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 241.908. Standardfehler: 0.827. Kosten je Aufgabe in USD: 0.034867. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 241.908. Standard error: 0.827. Cost per task in USD: 0.034867. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":49.152542372881356,"measurements":[{"observationId":"vals-swebench:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":76.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.152542372881356,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 519.829. Standardfehler: 1.906. Kosten je Aufgabe in USD: 0.27474. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 519.829. Standard error: 1.906. Cost per task in USD: 0.27474. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":45.28600464576074,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:0de67206-4d36-4d10-b8f6-cf37fa747a03","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=4451:suffix=Reasoning","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0de67206-4d36-4d10-b8f6-cf37fa747a03/kimi-k2-6/kimi-k2-6. Source model name: Kimi K2.6. Provider reported by source: Kimi. Model snapshot: Kimi K2.6 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.6 (Reasoning) (Release kimi-k2-6, Zeile kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Kimi K2.6 (Reasoning) (release kimi-k2-6, row kimi-k2-6, UUID 0de67206-4d36-4d10-b8f6-cf37fa747a03, reasoning=true, source reasoning-token configuration budget=4451, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":57.303,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.19047619047619,"participants":43,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 795.458. Standardfehler: 5.273. Kosten je Aufgabe in USD: 0.192786. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 795.458. Standard error: 5.273. Cost per task in USD: 0.192786. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":53.558,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.678571428571427,"participants":57,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 932.358. Standardfehler: 2.085. Kosten je Aufgabe in USD: 0.138449. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 932.358. Standard error: 2.085. Cost per task in USD: 0.138449. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":50,"measurements":[{"observationId":"vals-code-migration:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":27.768,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":63,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5398.566. Standardfehler: 4.144. Kosten je Aufgabe in USD: 5.112053. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 5398.566. Standard error: 4.144. Cost per task in USD: 5.112053. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":85.13513513513513,"measurements":[{"observationId":"vals-corp-fin-v2:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":66.744,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.13513513513513,"participants":75,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Top P: 0.95. Maximale Ausgabetoken: 48000. Latenz in Sekunden: 79.59. Standardfehler: 0.928. Kosten je Aufgabe in USD: 0.07328. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Top P: 0.95. Maximum output tokens: 48000. Latency in seconds: 79.59. Standard error: 0.928. Cost per task in USD: 0.07328. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 48000","en":"Top P: 0.95. Maximum output tokens: 48000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 48000","en":"Top P: 0.95. Maximum output tokens: 48000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":47.45762711864407,"measurements":[{"observationId":"vals-emb:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":57.855,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.45762711864407,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2387.544. Standardfehler: 2.739. Kosten je Aufgabe in USD: 2.195977. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2387.544. Standard error: 2.739. Cost per task in USD: 2.195977. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":31.666666666666668,"measurements":[{"observationId":"vals-finance-agent-v2:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":44.899,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1629.639. Standardfehler: 0.733. Kosten je Aufgabe in USD: 1.141587. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1629.639. Standard error: 0.733. Cost per task in USD: 1.141587. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":79.22077922077922,"measurements":[{"observationId":"vals-lcb:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":86.771,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.22077922077922,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 529.063. Standardfehler: 0.972. Kosten je Aufgabe in USD: 0.06543. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 529.063. Standard error: 0.972. Cost per task in USD: 0.06543. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":37.704918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.704918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1405.253. Standardfehler: 0.833. Kosten je Aufgabe in USD: 0.818741. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1405.253. Standard error: 0.833. Cost per task in USD: 0.818741. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":73.07692307692308,"measurements":[{"observationId":"vals-legal-bench:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.738,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.07692307692308,"participants":79,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 50.067. Standardfehler: 0.449. Kosten je Aufgabe in USD: 0.0065. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 50.067. Standard error: 0.449. Cost per task in USD: 0.0065. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":23.333333333333332,"measurements":[{"observationId":"vals-legal-research:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":15.865,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1853.133. Standardfehler: 2.539. Kosten je Aufgabe in USD: 1.520305. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1853.133. Standard error: 2.539. Cost per task in USD: 1.520305. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":28.35820895522388,"measurements":[{"observationId":"vals-medcode:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.142,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.35820895522388,"participants":68,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 305.421. Standardfehler: 2.041. Kosten je Aufgabe in USD: 0.041295. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 305.421. Standard error: 2.041. Cost per task in USD: 0.041295. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":35.294117647058826,"measurements":[{"observationId":"vals-medscribe:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":78.149,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.294117647058826,"participants":69,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 494.142. Standardfehler: 1.792. Kosten je Aufgabe in USD: 0.055962. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 494.142. Standard error: 1.792. Cost per task in USD: 0.055962. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":76.62337662337663,"measurements":[{"observationId":"vals-mmlu-pro:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.572,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.62337662337663,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 156.347. Standardfehler: 0.33. Kosten je Aufgabe in USD: 0.018214. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 156.347. Standard error: 0.33. Cost per task in USD: 0.018214. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"vals-mortgage-tax:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.818,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":53,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 109.76. Standardfehler: 0.93. Kosten je Aufgabe in USD: 0.02869. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 109.76. Standard error: 0.93. Cost per task in USD: 0.02869. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 4794.524. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 4794.524. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":77.58620689655173,"measurements":[{"observationId":"vals-sage:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":50.224,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.58620689655173,"participants":59,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 270.956. Standardfehler: 3.429. Kosten je Aufgabe in USD: 0.068198. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 270.956. Standard error: 3.429. Cost per task in USD: 0.068198. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":19.736842105263158,"measurements":[{"observationId":"vals-snap:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":56.631,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.736842105263158,"participants":39,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 595.296. Standardfehler: 1.289. Kosten je Aufgabe in USD: 0.906347. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 595.296. Standard error: 1.289. Cost per task in USD: 0.906347. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-tax-agent-bench:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":12.404,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":56,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 924.425. Standardfehler: 2.376. Kosten je Aufgabe in USD: 0.565511. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 924.425. Standard error: 2.376. Cost per task in USD: 0.565511. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":71.05263157894737,"measurements":[{"observationId":"vals-tax-eval-v2:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.652,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 267.484. Standardfehler: 0.846. Kosten je Aufgabe in USD: 0.029274. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 267.484. Standard error: 0.846. Cost per task in USD: 0.029274. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":37.5,"measurements":[{"observationId":"vals-vibe-code-bench:overall:kimi/kimi-k2.6","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":37.891,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.5,"participants":73,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.6. Name des Quellmodells: Kimi K2.6. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2967.768. Standardfehler: 4.912. Kosten je Aufgabe in USD: 1.925351. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.6. Source model name: Kimi K2.6. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2967.768. Standard error: 4.912. Cost per task in USD: 1.925351. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":34,"totalFamilies":71,"sourceCount":3,"coverage":0.4219954648526078,"lowerBound":20.917179678536005,"upperBound":78.71763319327522,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":7,"totalFamilies":15,"sourceCount":3,"coverage":0.42222222222222233,"lowerBound":19.123901124248082,"upperBound":76.90167890202585,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":28.326150742710436,"upperBound":85.46900788556758,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":71.23851757998099,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":47.49234505332067,"upperBound":80.825678386654,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":18.380237648530333,"upperBound":76.71357098186365,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.111111111111112,"upperBound":86.11111111111111,"provisional":true,"configurationsDiffer":true},"finance":{"score":51.456438987661116,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":44.10551913228095,"upperBound":58.391233417995245,"provisional":true,"configurationsDiffer":true},"legal":{"score":44.7050581476811,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":33.52879361076082,"upperBound":58.52879361076082,"provisional":true,"configurationsDiffer":true},"health":{"score":31.826163301141353,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":21.217442200760903,"upperBound":54.55077553409424,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":51.786980749746704,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":25.893490374873352,"upperBound":75.89349037487335,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":9.868421052631579,"upperBound":59.868421052631575,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":38.793103448275865,"upperBound":88.79310344827587,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2.7-code","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":858.75,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":79.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 79,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 79.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":54.8780487804878,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":24.454304615216714,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.8780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 24,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 24.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":50.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":11.200000000000001,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 11,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 11.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":47.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":1037.51,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":47.82608695652174,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":63.1292517006803,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":47.82608695652174,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 63,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 63.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":-10.216666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":51.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":35.032437442075995,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 35,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 35.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":60.16949152542373,"measurements":[{"observationId":"vals-swebench:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":78.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.16949152542373,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Latenz in Sekunden: 896.71. Standardfehler: 1.848. Kosten je Aufgabe in USD: 0.265276. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Latency in seconds: 896.71. Standard error: 1.848. Cost per task in USD: 0.265276. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":null,"scoreRunConfiguration":null,"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":49.096254355400696,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:8d0cb231-7303-452c-9923-a9620b948475","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2229:suffix=unspecified","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Kimi. Modellstand: Kimi K2.7 Code. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8d0cb231-7303-452c-9923-a9620b948475/kimi-k2-7-code/kimi-k2-7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Kimi. Model snapshot: Kimi K2.7 Code. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K2.7 Code (Release kimi-k2-7-code, Zeile kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Kimi K2.7 Code (release kimi-k2-7-code, row kimi-k2-7-code, UUID 8d0cb231-7303-452c-9923-a9620b948475, reasoning=true, source reasoning-token configuration budget=2229, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":67.041,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.214285714285715,"participants":57,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 784.159. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.256503. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 784.159. Standard error: 0.375. Cost per task in USD: 0.256503. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":41.935483870967744,"measurements":[{"observationId":"vals-code-migration:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":25.392,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.935483870967744,"participants":63,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 5945.669. Standardfehler: 4.161. Kosten je Aufgabe in USD: 5.040913. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 5945.669. Standard error: 4.161. Cost per task in USD: 5.040913. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":46.753246753246756,"measurements":[{"observationId":"vals-lcb:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.05,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.753246753246756,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 214.178. Standardfehler: 1.071. Kosten je Aufgabe in USD: 0.027349. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 214.178. Standard error: 1.071. Cost per task in USD: 0.027349. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 3860.332. Standardfehler: 0. Kosten je Aufgabe in USD: 3.185154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 3860.332. Standard error: 0. Cost per task in USD: 3.185154. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":41.37931034482759,"measurements":[{"observationId":"vals-skillsbench:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":50.036,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.37931034482759,"participants":30,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 803.827. Standardfehler: 4.237. Kosten je Aufgabe in USD: 0.443192. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 803.827. Standard error: 4.237. Cost per task in USD: 0.443192. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":41.666666666666664,"measurements":[{"observationId":"vals-vibe-code-bench:overall:kimi/kimi-k2.7-code","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":47.211,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.666666666666664,"participants":73,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k2.7-code. Name des Quellmodells: Kimi K2.7 Code. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 10318.246. Standardfehler: 5.191. Kosten je Aufgabe in USD: 3.835806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k2.7-code. Source model name: Kimi K2.7 Code. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768. Latency in seconds: 10318.246. Standard error: 5.191. Cost per task in USD: 3.835806. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 32768","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":15,"totalFamilies":71,"sourceCount":2,"coverage":0.11201814058956916,"lowerBound":5.289600646104611,"upperBound":94.0877865871477,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.34444444444444455,"lowerBound":14.510718347259617,"upperBound":80.06627390281515,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":19.294425087108014,"upperBound":83.5801393728223,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":14.634146341463417,"upperBound":81.30081300813009,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":2,"coverage":0.3333333333333333,"lowerBound":16.049901878329127,"upperBound":82.7165685449958,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":9.56521739130435,"upperBound":89.56521739130434,"provisional":true,"configurationsDiffer":true}}},{"modelId":"kimi-k2.8-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"kimi-k3","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":38.63636363636363,"measurements":[{"observationId":"research:kimi-k3:65","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9722222222222222,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":38.63636363636363,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: A33om9MTYMC7xYeLv9X7Ha. Startzeit: 2026-07-16T21:59:40.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9722222222222222 und stderr=0.011158359139033182. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: A33om9MTYMC7xYeLv9X7Ha. Started at 2026-07-16T21:59:40.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9722222222222222 and stderr=0.011158359139033182. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":43.421052631578945,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":38.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.421052631578945,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 38,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 38.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1499.38,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":88.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 88,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 88.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":92.3076923076923,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":41.2979351032448,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.3076923076923,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 41,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 41.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":58.272977800152034,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 58,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 58.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":43.6099333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.33333333333333,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 43,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 43.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":22,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 22,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 22.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1532.59,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":80,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":47.6930320150659,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 47,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 47.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":19.7,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-chess-puzzles","area":"reasoning","label":{"de":"Chess Puzzles","en":"Chess Puzzles"},"weight":0.01020408163265306,"coverage":1,"score":26.19047619047619,"measurements":[{"observationId":"research:kimi-k3:61","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-chess-puzzles","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","retrievedAt":"2026-10-19","value":0.36,"unit":"ratio","metric":"exact","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":26.19047619047619,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: Chess Puzzles. Task-Version: 2.0.0. Lauf-ID: nzQV9YvictnDF2EwxqTJQs. Startzeit: 2026-10-13T17:13:35.000Z. Ursprüngliche Task-Bezeichnung: Chess Puzzles. Die CSV-Zeile meldet mean_score=0.36 und stderr=0.048241815132442176. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: Chess Puzzles. Task version: 2.0.0. Run ID: nzQV9YvictnDF2EwxqTJQs. Started at 2026-10-13T17:13:35.000Z. Original task label: Chess Puzzles. The published row reports mean_score=0.36 and stderr=0.048241815132442176. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"Chess Puzzles\",\"Chess Puzzles\",\"2.0.0\",\"exact\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"epoch-simpleqa-verified","area":"knowledge","label":{"de":"SimpleQA Verified","en":"SimpleQA Verified"},"weight":0.023809523809523808,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"research:kimi-k3:62","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-simpleqa-verified","benchmarkVersion":"1.2.0","comparisonCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","retrievedAt":"2026-10-19","value":0.506,"unit":"ratio","metric":"simpleqa_scorer","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":44.44444444444444,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: SimpleQA Verified. Task-Version: 1.2.0. Lauf-ID: mZLWt2JSYvJyGhMvji7CT8. Startzeit: 2026-08-27T19:36:12.000Z. Ursprüngliche Task-Bezeichnung: SimpleQA Verified (anti-abstention). Die CSV-Zeile meldet mean_score=0.506 und stderr=0.015818160898606836. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: SimpleQA Verified. Task version: 1.2.0. Run ID: mZLWt2JSYvJyGhMvji7CT8. Started at 2026-08-27T19:36:12.000Z. Original task label: SimpleQA Verified (anti-abstention). The published row reports mean_score=0.506 and stderr=0.015818160898606836. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"SimpleQA Verified\",\"SimpleQA Verified (anti-abstention)\",\"1.2.0\",\"simpleqa_scorer\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":1,"score":60.60606060606061,"measurements":[{"observationId":"research:kimi-k3:55","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tier-4-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.3902439024390244,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":66.66666666666667,"participants":19,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tier-4-v2-Private. Task-Version: 2.0.0. Lauf-ID: gZERSAQwxBAevQzBsWfUie. Startzeit: 2026-07-17T01:48:42.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tier-4-v2-Private. Die CSV-Zeile meldet mean_score=0.3902439024390244 und stderr=0.07712872341874095. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tier-4-v2-Private. Task version: 2.0.0. Run ID: gZERSAQwxBAevQzBsWfUie. Started at 2026-07-17T01:48:42.000Z. Original task label: FrontierMath-Tier-4-v2-Private. The published row reports mean_score=0.3902439024390244 and stderr=0.07712872341874095. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tier-4-v2-Private\",\"FrontierMath-Tier-4-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"research:kimi-k3:54","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.721830985915493,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":54.54545454545455,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: bcntamvGsSH6QgETdoaSph. Startzeit: 2026-07-17T01:48:42.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.721830985915493 und stderr=0.026636607935112643. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: bcntamvGsSH6QgETdoaSph. Started at 2026-07-17T01:48:42.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.721830985915493 and stderr=0.026636607935112643. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":1,"score":68.62468671679198,"measurements":[{"observationId":"research:kimi-k3:58","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-gpqa-diamond","benchmarkVersion":"1.0.11","comparisonCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","retrievedAt":"2026-10-19","value":0.9311868686868687,"unit":"ratio","metric":"choice","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":52.38095238095238,"participants":22,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: kimi-k3_max. Epoch-Modellbezeichnung: Kimi K3. Anzeigenamenfeld: Kimi K3 (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: GPQA diamond. Task-Version: 1.0.11. Lauf-ID: B8MaPUe3EYwUZASsNqE7C9. Startzeit: 2026-07-16T21:59:40.000Z. Ursprüngliche Task-Bezeichnung: GPQA diamond. Die CSV-Zeile meldet mean_score=0.9311868686868687 und stderr=0.014937225364194858. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: kimi-k3_max; Epoch model label: Kimi K3; Display label field: Kimi K3 (max); Reasoning setting encoded in the alias: max. Task ID: GPQA diamond. Task version: 1.0.11. Run ID: B8MaPUe3EYwUZASsNqE7C9. Started at 2026-07-16T21:59:40.000Z. Original task label: GPQA diamond. The published row reports mean_score=0.9311868686868687 and stderr=0.014937225364194858. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"GPQA diamond\",\"GPQA diamond\",\"1.0.11\",\"choice\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-gpqa:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":92.929,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.86842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 122.425. Standardfehler: 1.309. Kosten je Aufgabe in USD: 0.088491. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 122.425. Standard error: 1.309. Cost per task in USD: 0.088491. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":46.8952734012975,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 46,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 46.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":75,"measurements":[{"observationId":"lmarena:webdev:overall:kimi-k3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1655.2951601625216,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":75,"participants":37,"methodology":{"de":"Arena-Bewertung aus 18.644 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/kimi-k3-max. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Modellstand: kimi-k3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 18,644 published pairwise votes.. Source model ID: lmarena/kimi-k3-max. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Model snapshot: kimi-k3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 18.644 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 18,644 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 18.644 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 18,644 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":80.55555555555556,"measurements":[{"observationId":"lmarena:overall:kimi-k3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1488.0950820118865,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":80.55555555555556,"participants":37,"methodology":{"de":"Arena-Bewertung aus 32.036 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/kimi-k3-max. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Modellstand: kimi-k3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 32,036 published pairwise votes.. Source model ID: lmarena/kimi-k3-max. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Model snapshot: kimi-k3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 32.036 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 32,036 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 32.036 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 32,036 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":77.75455927051672,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":80.52023121387279,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.78571428571429,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.15,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":78.72340425531915,"participants":48,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 106.516. Standardfehler: 0.777. Kosten je Aufgabe in USD: 0.059115. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 106.516. Standard error: 0.777. Cost per task in USD: 0.059115. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":89.83050847457628,"measurements":[{"observationId":"vals-swebench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":93.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.83050847457628,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 619.492. Standardfehler: 1.111. Kosten je Aufgabe in USD: 0.759959. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 619.492. Standard error: 1.111. Cost per task in USD: 0.759959. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":67.59440625294285,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:f7d2fc3e-1f7b-405f-818c-07952a4af78f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":12.6262626262626,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":70.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Kimi. Modellstand: Kimi K3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 12,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/f7d2fc3e-1f7b-405f-818c-07952a4af78f/kimi-k3/kimi-k3. Source model name: Kimi K3. Provider reported by source: Kimi. Model snapshot: Kimi K3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 12.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Kimi K3 (Max) (Release kimi-k3, Zeile kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Kimi K3 (Max) (release kimi-k3, row kimi-k3, UUID f7d2fc3e-1f7b-405f-818c-07952a4af78f, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":80.899,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.71428571428571,"participants":57,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 608.575. Standardfehler: 0.649. Kosten je Aufgabe in USD: 0.341034. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 608.575. Standard error: 0.649. Cost per task in USD: 0.341034. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":17.172,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.891891891891895,"participants":38,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 11353.52. Standardfehler: 0.505. Kosten je Aufgabe in USD: 12.478176. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 11353.52. Standard error: 0.505. Cost per task in USD: 12.478176. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":14.516129032258064,"measurements":[{"observationId":"vals-terminal-bench-science:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":1.429,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.516129032258064,"participants":32,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 17349.404. Standardfehler: 1.429. Kosten je Aufgabe in USD: 21.301409. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 17349.404. Standard error: 1.429. Cost per task in USD: 21.301409. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":63.1578947368421,"measurements":[{"observationId":"vals-biomysterybench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":72.593,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.1578947368421,"participants":20,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 295.415. Standardfehler: 1.614. Kosten je Aufgabe in USD: 1.060934. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 295.415. Standard error: 1.614. Cost per task in USD: 1.060934. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":32.25806451612903,"measurements":[{"observationId":"vals-code-migration:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":16.099,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.25806451612903,"participants":63,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 15450.785. Standardfehler: 4.101. Kosten je Aufgabe in USD: 13.871104. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 15450.785. Standard error: 4.101. Cost per task in USD: 13.871104. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":97.29729729729729,"measurements":[{"observationId":"vals-corp-fin-v2:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":71.562,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.29729729729729,"participants":75,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 37.99. Standardfehler: 0.888. Kosten je Aufgabe in USD: 0.063819. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 37.99. Standard error: 0.888. Cost per task in USD: 0.063819. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":84.84848484848484,"measurements":[{"observationId":"vals-cyberbench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":75.238,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.84848484848484,"participants":34,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 1844.34. Standardfehler: 5.562. Kosten je Aufgabe in USD: 2.12561. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 262144. Latency in seconds: 1844.34. Standard error: 5.562. Cost per task in USD: 2.12561. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":71.1864406779661,"measurements":[{"observationId":"vals-emb:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":66.677,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":71.1864406779661,"participants":60,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 741.42. Standardfehler: 2.873. Kosten je Aufgabe in USD: 3.074655. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 741.42. Standard error: 2.873. Cost per task in USD: 3.074655. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":60,"measurements":[{"observationId":"vals-finance-agent-v2:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":53.114,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 285.753. Standardfehler: 0.374. Kosten je Aufgabe in USD: 1.913224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 285.753. Standard error: 0.374. Cost per task in USD: 1.913224. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-ioi:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":48.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":34,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 15508.583. Standardfehler: 9.82. Kosten je Aufgabe in USD: 14.165034. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 15508.583. Standard error: 9.82. Cost per task in USD: 14.165034. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"vals-lcb:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.185,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 200.263. Standardfehler: 0.971. Kosten je Aufgabe in USD: 0.103719. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 262144. Latency in seconds: 200.263. Standard error: 0.971. Cost per task in USD: 0.103719. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":90.98360655737704,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":12.917,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.98360655737704,"participants":62,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 837.646. Standardfehler: 2.679. Kosten je Aufgabe in USD: 3.829773. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 837.646. Standard error: 2.679. Cost per task in USD: 3.829773. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"vals-legal-bench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":86.212,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":79,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 7.106. Standardfehler: 0.412. Kosten je Aufgabe in USD: 0.007921. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 7.106. Standard error: 0.412. Cost per task in USD: 0.007921. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":80,"measurements":[{"observationId":"vals-legal-research:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":46.154,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80,"participants":61,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 760.122. Standardfehler: 3.465. Kosten je Aufgabe in USD: 3.469601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 760.122. Standard error: 3.465. Cost per task in USD: 3.469601. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":74.6268656716418,"measurements":[{"observationId":"vals-medcode:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.357,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.6268656716418,"participants":68,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 36.994. Standardfehler: 2.202. Kosten je Aufgabe in USD: 0.093857. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 36.994. Standard error: 2.202. Cost per task in USD: 0.093857. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":85.29411764705883,"measurements":[{"observationId":"vals-medscribe:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":88.048,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.29411764705883,"participants":69,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 38.241. Standardfehler: 1.976. Kosten je Aufgabe in USD: 0.110262. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 38.241. Standard error: 1.976. Cost per task in USD: 0.110262. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":77.92207792207792,"measurements":[{"observationId":"vals-mmlu-pro:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.973,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.92207792207792,"participants":78,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: max. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 38.718. Standardfehler: 0.321. Kosten je Aufgabe in USD: 0.021327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: max. Maximum output tokens: 262144. Latency in seconds: 38.718. Standard error: 0.321. Cost per task in USD: 0.021327. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":51.92307692307692,"measurements":[{"observationId":"vals-mortgage-tax:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":66.336,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.92307692307692,"participants":53,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 78.943. Standardfehler: 0.909. Kosten je Aufgabe in USD: 0.038981. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 78.943. Standard error: 0.909. Cost per task in USD: 0.038981. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":83.6734693877551,"measurements":[{"observationId":"vals-programbench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":2,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.6734693877551,"participants":50,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 20494.929. Standardfehler: 0.992. Kosten je Aufgabe in USD: 70.475273. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 20494.929. Standard error: 0.992. Cost per task in USD: 70.475273. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"vals-proof-bench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":87,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":40,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 2070.511. Standardfehler: 3.38. Kosten je Aufgabe in USD: 1.646802. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Maximum output tokens: 262144. Latency in seconds: 2070.511. Standard error: 3.38. Cost per task in USD: 1.646802. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":94.82758620689656,"measurements":[{"observationId":"vals-sage:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":52.783,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.82758620689656,"participants":59,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 47.3. Standardfehler: 3.414. Kosten je Aufgabe in USD: 0.163375. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 47.3. Standard error: 3.414. Cost per task in USD: 0.163375. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":81.57894736842105,"measurements":[{"observationId":"vals-snap:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":68.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.57894736842105,"participants":39,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 438.672. Standardfehler: 1.211. Kosten je Aufgabe in USD: 0.913402. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 438.672. Standard error: 1.211. Cost per task in USD: 0.913402. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"vals-swebench-multimodal:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":35.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":14,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 1280.589. Standardfehler: 2.191. Kosten je Aufgabe in USD: 1.501383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 1280.589. Standard error: 2.191. Cost per task in USD: 1.501383. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"vals-tax-agent-bench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":33.368,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":56,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 2415.159. Standardfehler: 3.386. Kosten je Aufgabe in USD: 2.790127. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 2415.159. Standard error: 3.386. Cost per task in USD: 2.790127. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":93.42105263157895,"measurements":[{"observationId":"vals-tax-eval-v2:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.716,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.42105263157895,"participants":77,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 144.708. Standardfehler: 0.841. Kosten je Aufgabe in USD: 0.078156. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Maximum output tokens: 262144. Latency in seconds: 144.708. Standard error: 0.841. Cost per task in USD: 0.078156. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 262144","en":"Temperature: 1. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":47.61904761904762,"measurements":[{"observationId":"vals-vcb-1-100:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":18.244,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":47.61904761904762,"participants":22,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 6487.278. Standardfehler: 3.92. Kosten je Aufgabe in USD: 9.39737. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 6487.278. Standard error: 3.92. Cost per task in USD: 9.39737. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":84.72222222222223,"measurements":[{"observationId":"vals-vibe-code-bench:overall:kimi/kimi-k3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":84.968,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.72222222222223,"participants":73,"methodology":{"de":"Kennung des Quellmodells: kimi/kimi-k3. Name des Quellmodells: Kimi K3. Anbieter laut Quelle: Moonshot AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144. Latenz in Sekunden: 999.021. Standardfehler: 2.746. Kosten je Aufgabe in USD: 10.007576. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: kimi/kimi-k3. Source model name: Kimi K3. Provider reported by source: Moonshot AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144. Latency in seconds: 999.021. Standard error: 2.746. Cost per task in USD: 10.007576. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 262144","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Top P: 0.95. Maximum output tokens: 262144"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":75.21995369988245,"coveredFamilies":46,"totalFamilies":71,"sourceCount":4,"coverage":0.5992063492063494,"lowerBound":45.0722738439772,"upperBound":85.15163892334226,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":57.534456543800786,"coveredFamilies":12,"totalFamilies":15,"sourceCount":3,"coverage":0.7222222222222222,"lowerBound":41.55266305941168,"upperBound":69.33044083718946,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":71.07003620161515,"coveredFamilies":5,"totalFamilies":7,"sourceCount":3,"coverage":0.6428571428571428,"lowerBound":45.687880415324024,"upperBound":81.40216612960975,"provisional":true,"configurationsDiffer":true},"math":{"score":58.721833721833725,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":35.23310023310024,"upperBound":75.23310023310023,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":66.39859688640176,"coveredFamilies":3,"totalFamilies":3,"sourceCount":3,"coverage":1,"lowerBound":66.39859688640176,"upperBound":66.39859688640176,"provisional":true,"configurationsDiffer":true},"agentic":{"score":77.21495342483789,"coveredFamilies":6,"totalFamilies":12,"sourceCount":1,"coverage":0.5,"lowerBound":38.60747671241894,"upperBound":88.60747671241894,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":28.742690058479532,"upperBound":88.74269005847952,"provisional":true,"configurationsDiffer":true},"finance":{"score":75.94100822468351,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":65.09229276401445,"upperBound":79.37800704972874,"provisional":true,"configurationsDiffer":true},"legal":{"score":85.19966372425387,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":63.8997477931904,"upperBound":88.8997477931904,"provisional":true,"configurationsDiffer":true},"health":{"score":79.96049165935031,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":53.30699443956688,"upperBound":86.6403277729002,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":84.84848484848484,"upperBound":84.84848484848484,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":19.43863981762918,"upperBound":94.43863981762918,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":40.78947368421053,"upperBound":90.78947368421052,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":47.41379310344828,"upperBound":97.41379310344828,"provisional":true,"configurationsDiffer":true}}},{"modelId":"koa","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"kolibri-1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"laguna-s-2.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"laguna-xs-2.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ling-3-0-flash-fin","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ling-3-0-flash-vl","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ling-3-0-tiny","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ling-3-1-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ling-3.0-flash","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":797.62,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":33.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":73,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 73,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 73.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":26.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":3.219648917147413,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 3,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 3.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":5.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 5,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 5.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":39.02439024390244,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":947.38,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.02439024390244,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":35.36585365853659,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-17.883333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.36585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":52.63157894736842,"measurements":[{"observationId":"vals-gpqa:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":84.848,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.63157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 43.513. Standardfehler: 2.197. Kosten je Aufgabe in USD: 0.003327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 43.513. Standard error: 2.197. Cost per task in USD: 0.003327. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":34.146341463414636,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":23.6793327154773,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.146341463414636,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 23,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 23.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":13.559322033898304,"measurements":[{"observationId":"vals-swebench:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":65.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.559322033898304,"participants":60,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 195.978. Standardfehler: 2.132. Kosten je Aufgabe in USD: 0.031792. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 195.978. Standard error: 2.132. Cost per task in USD: 0.031792. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":18.172909407665507,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:e33dc369-ae81-40e0-8c4e-28b002d02197","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: InclusionAI. Modellstand: Ling 3.0 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e33dc369-ae81-40e0-8c4e-28b002d02197/ling-3-0-flash/ling-3-0-flash. Source model name: Ling 3.0 Flash. Provider reported by source: InclusionAI. Model snapshot: Ling 3.0 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ling 3.0 Flash (Release ling-3-0-flash, Zeile ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ling 3.0 Flash (release ling-3-0-flash, row ling-3-0-flash, UUID e33dc369-ae81-40e0-8c4e-28b002d02197, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":50.187,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.642857142857142,"participants":57,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 593.826. Standardfehler: 2.456. Kosten je Aufgabe in USD: 0.067399. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 593.826. Standard error: 2.456. Cost per task in USD: 0.067399. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"vals-code-migration:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":63,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 17934.764. Standardfehler: 0. Kosten je Aufgabe in USD: 0.060112. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 17934.764. Standard error: 0. Cost per task in USD: 0.060112. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":55.4054054054054,"measurements":[{"observationId":"vals-corp-fin-v2:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.927,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.4054054054054,"participants":75,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 12.451. Standardfehler: 0.957. Kosten je Aufgabe in USD: 0.006376. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 12.451. Standard error: 0.957. Cost per task in USD: 0.006376. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":10.169491525423728,"measurements":[{"observationId":"vals-emb:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":25.006,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.169491525423728,"participants":60,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 735.485. Standardfehler: 2.462. Kosten je Aufgabe in USD: 0.179925. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 735.485. Standard error: 2.462. Cost per task in USD: 0.179925. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":8.333333333333334,"measurements":[{"observationId":"vals-finance-agent-v2:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":30.314,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.333333333333334,"participants":61,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 157.02. Standardfehler: 0.275. Kosten je Aufgabe in USD: 0.05047. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 157.02. Standard error: 0.275. Cost per task in USD: 0.05047. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":59.74025974025974,"measurements":[{"observationId":"vals-lcb:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":83.988,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.74025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 60.397. Standardfehler: 1.038. Kosten je Aufgabe in USD: 0.003019. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 60.397. Standard error: 1.038. Cost per task in USD: 0.003019. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":31.147540983606557,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.25,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.147540983606557,"participants":62,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 161.487. Standardfehler: 0.833. Kosten je Aufgabe in USD: 0.046313. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 161.487. Standard error: 0.833. Cost per task in USD: 0.046313. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":29.487179487179485,"measurements":[{"observationId":"vals-legal-bench:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.69,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.487179487179485,"participants":79,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3.896. Standardfehler: 0.515. Kosten je Aufgabe in USD: 0.000172. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3.896. Standard error: 0.515. Cost per task in USD: 0.000172. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"vals-legal-research:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":61,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 211.833. Standardfehler: 0. Kosten je Aufgabe in USD: 0.036023. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 211.833. Standard error: 0. Cost per task in USD: 0.036023. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":10.447761194029852,"measurements":[{"observationId":"vals-medcode:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":32.275,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.447761194029852,"participants":68,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 9.647. Standardfehler: 1.908. Kosten je Aufgabe in USD: 0.001645. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 9.647. Standard error: 1.908. Cost per task in USD: 0.001645. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":45.588235294117645,"measurements":[{"observationId":"vals-medscribe:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":80.904,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.588235294117645,"participants":69,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 12.121. Standardfehler: 2.039. Kosten je Aufgabe in USD: 0.001358. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 12.121. Standard error: 2.039. Cost per task in USD: 0.001358. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":31.16883116883117,"measurements":[{"observationId":"vals-mmlu-pro:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":82.008,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.16883116883117,"participants":78,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 20.073. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.000862. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 20.073. Standard error: 0.375. Cost per task in USD: 0.000862. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":6.896551724137931,"measurements":[{"observationId":"vals-skillsbench:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":27.946,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.896551724137931,"participants":30,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 292.431. Standardfehler: 3.76. Kosten je Aufgabe in USD: 0.060905. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 292.431. Standard error: 3.76. Cost per task in USD: 0.060905. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":2.6315789473684212,"measurements":[{"observationId":"vals-snap:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":50.744,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.6315789473684212,"participants":39,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 275.254. Standardfehler: 1.3. Kosten je Aufgabe in USD: 0.049736. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 275.254. Standard error: 1.3. Cost per task in USD: 0.049736. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":25,"measurements":[{"observationId":"vals-tax-eval-v2:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":70.646,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":77,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 7.54. Standardfehler: 0.895. Kosten je Aufgabe in USD: 0.000664. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 7.54. Standard error: 0.895. Cost per task in USD: 0.000664. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":6.944444444444445,"measurements":[{"observationId":"vals-vibe-code-bench:overall:ant/ling-3.0-flash-2607","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":2.914,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.944444444444445,"participants":73,"methodology":{"de":"Kennung des Quellmodells: ant/ling-3.0-flash-2607. Name des Quellmodells: Ling 3.0 Flash. Anbieter laut Quelle: Ant. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1005.598. Standardfehler: 2.072. Kosten je Aufgabe in USD: 0.292199. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: ant/ling-3.0-flash-2607. Source model name: Ling 3.0 Flash. Provider reported by source: Ant. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1005.598. Standard error: 2.072. Cost per task in USD: 0.292199. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":25,"totalFamilies":71,"sourceCount":2,"coverage":0.29960317460317465,"lowerBound":7.277414725314028,"upperBound":77.31709726499656,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.2777777777777779,"lowerBound":5.705309209339802,"upperBound":77.92753143156202,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":14.99633229415001,"upperBound":72.13918943700715,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":33.26734241368388,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":22.17822827578925,"upperBound":55.51156160912259,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":2,"coverage":0.3333333333333333,"lowerBound":9.721054107092794,"upperBound":76.38772077375947,"provisional":true,"configurationsDiffer":true},"finance":{"score":24.727057566040614,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":14.129747180594636,"upperBound":56.986890037737496,"provisional":true,"configurationsDiffer":true},"legal":{"score":20.211573490262015,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":15.15868011769651,"upperBound":40.15868011769651,"provisional":true,"configurationsDiffer":true},"health":{"score":28.01799824407375,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":18.678665496049167,"upperBound":52.011998829382506,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":1.3157894736842106,"upperBound":51.31578947368421,"provisional":true,"configurationsDiffer":true}}},{"modelId":"llada-2.2-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-3.1-70b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-3.1-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-3.2-1b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-3.2-3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-3.3-70b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"llama-4-maverick","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":50,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 50,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 50.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":3.658536585365854,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.2121918776569101,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.658536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-274.88,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":35.507246376811594,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":42.9931972789116,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.507246376811594,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 43,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 43.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-41.85,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.91195551436515,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":23.214285714285715,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":62.1387283236994,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.214285714285715,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 62,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 62.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-multinrc-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":8.44,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 8,44 % ± 1,68. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-bc7fb4bc0d9a. Prüfsumme der Quelldaten: bc7fb4bc0d9ad4757d46043b7c57b0b056967f2b0062249a99a3f0b616b95e23. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 8.44% ± 1.68. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-bc7fb4bc0d9a. Source payload hash: bc7fb4bc0d9ad4757d46043b7c57b0b056967f2b0062249a99a3f0b616b95e23. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-prbench-finance-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":22.36,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 22,36 % ± 0,42. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-70cdca115a0d. Prüfsumme der Quelldaten: 70cdca115a0da049a285a4ae00fcd400f49e1eea46970bf26fcd4c3374930b5c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 22.36% ± 0.42. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-70cdca115a0d. Source payload hash: 70cdca115a0da049a285a4ae00fcd400f49e1eea46970bf26fcd4c3374930b5c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-prbench-legal-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":24.84,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 24,84 % ± 0,9. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-f4e7b1dd0a3e. Prüfsumme der Quelldaten: f4e7b1dd0a3ecd0a1d7d7b08e33377b62d408b58c2f0c61a413a4cfec0d57c2a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 24.84% ± 0.9. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-f4e7b1dd0a3e. Source payload hash: f4e7b1dd0a3ecd0a1d7d7b08e33377b62d408b58c2f0c61a413a4cfec0d57c2a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Maverick. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/922c69c7-9037-43c6-8bcf-a1c555e7f3eb/llama-4-maverick/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Model snapshot: Llama 4 Maverick. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Maverick (Release llama-4-maverick, Zeile llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Maverick (release llama-4-maverick, row llama-4-maverick, UUID 922c69c7-9037-43c6-8bcf-a1c555e7f3eb, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"curated:scale-tutorbench-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":40.2,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":11.11111111111111,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 40,2 % ± 1. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-642ef8e45495. Prüfsumme der Quelldaten: 642ef8e45495e730a3e2f2ddc114f60328e5c0c6da76f877e3b1435841792058. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 40.2% ± 1. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-642ef8e45495. Source payload hash: 642ef8e45495e730a3e2f2ddc114f60328e5c0c6da76f877e3b1435841792058. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":7.6923076923076925,"measurements":[{"observationId":"curated:scale-vista-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":38.33,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":7.6923076923076925,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 38,33 % ± 0,55. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-ca07b51d9088. Prüfsumme der Quelldaten: ca07b51d90881a89b710581569dec80da9f3d102bf237f2da6dbe4ad5c91b720. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 38.33% ± 0.55. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-ca07b51d9088. Source payload hash: ca07b51d90881a89b710581569dec80da9f3d102bf237f2da6dbe4ad5c91b720. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"curated:scale-visual-toolbench-llama-4-maverick","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":1.41,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":0,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/llama-4-maverick. Name des Quellmodells: Llama 4 Maverick. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 1,41 % ± 0,25. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-b6f662a26398. Prüfsumme der Quelldaten: b6f662a2639828660d3c7ee001d8c431eb80967131af7ab01b3e852f4e8f899d. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/llama-4-maverick. Source model name: Llama 4 Maverick. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 1.41% ± 0.25. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-b6f662a26398. Source payload hash: b6f662a2639828660d3c7ee001d8c431eb80967131af7ab01b3e852f4e8f899d. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":16,"totalFamilies":71,"sourceCount":2,"coverage":0.2024659863945578,"lowerBound":2.035891645112786,"upperBound":81.789293005657,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":8.362369337979095,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5,"lowerBound":4.181184668989547,"upperBound":54.181184668989545,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":5.691056910569106,"upperBound":72.35772357723577,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":0.5589430894308943,"upperBound":75.5589430894309,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":7.101449275362319,"upperBound":87.10144927536231,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":0,"upperBound":85.71428571428572,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":0,"upperBound":75,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":7.719780219780221,"coveredFamilies":3,"totalFamilies":4,"sourceCount":2,"coverage":0.625,"lowerBound":4.824862637362638,"upperBound":42.324862637362635,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":5.555555555555555,"upperBound":55.55555555555556,"provisional":true,"configurationsDiffer":true}}},{"modelId":"llama-4-scout","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":27.700000000000003,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 27,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 27.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":0.6097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.19111706206154658,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0.6097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":1.829268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.829268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":2.4390243902439024,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-162.86,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.4390243902439024,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":26.08695652173913,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":39.5238095238095,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.08695652173913,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 39,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 39.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-52.15,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":5.2631578947368425,"measurements":[{"observationId":"vals-gpqa:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":46.97,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.2631578947368425,"participants":77,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 12.092. Standardfehler: 2.491. Kosten je Aufgabe in USD: 0.000654. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 12.092. Standard error: 2.491. Cost per task in USD: 0.000654. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":1.2195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":3.7800000000000002,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.2195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 3,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 3.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":7.484802431610942,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":52.9479768786127,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.714285714285714,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 52,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 52.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":58.752,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.25531914893617,"participants":48,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 10.506. Standardfehler: 1.183. Kosten je Aufgabe in USD: 0.000835. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 10.506. Standard error: 1.183. Cost per task in USD: 0.000835. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:adf9a85e-abc3-4f28-937b-db6655cc5238","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Modellstand: Llama 4 Scout. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/adf9a85e-abc3-4f28-937b-db6655cc5238/llama-4-scout/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Model snapshot: Llama 4 Scout. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Llama 4 Scout (Release llama-4-scout, Zeile llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Llama 4 Scout (release llama-4-scout, row llama-4-scout, UUID adf9a85e-abc3-4f28-937b-db6655cc5238, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":8.108108108108109,"measurements":[{"observationId":"vals-corp-fin-v2:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":46.776,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.108108108108109,"participants":75,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 9.697. Standardfehler: 0.98. Kosten je Aufgabe in USD: 0.011639. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Maximum output tokens: 16384. Latency in seconds: 9.697. Standard error: 0.98. Cost per task in USD: 0.011639. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":6.4935064935064934,"measurements":[{"observationId":"vals-lcb:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":38.541,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.4935064935064934,"participants":78,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 16.762. Standardfehler: 1.016. Kosten je Aufgabe in USD: 0.000615. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 16.762. Standard error: 1.016. Cost per task in USD: 0.000615. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":12.820512820512821,"measurements":[{"observationId":"vals-legal-bench:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":72.036,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.820512820512821,"participants":79,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 0.392. Standardfehler: 0.479. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Maximum output tokens: 16384. Latency in seconds: 0.392. Standard error: 0.479. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":18.181818181818183,"measurements":[{"observationId":"vals-math500:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":79.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.181818181818183,"participants":23,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 10.523. Standardfehler: 1.815. Kosten je Aufgabe in USD: 0.00043. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 16384. Latency in seconds: 10.523. Standard error: 1.815. Cost per task in USD: 0.00043. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 16384","en":"Temperature: 0. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 16384","en":"Temperature: 0. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"vals-medcode:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":23.311,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":68,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 10.736. Standardfehler: 1.749. Kosten je Aufgabe in USD: 0.002176. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 10.736. Standard error: 1.749. Cost per task in USD: 0.002176. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":2.5641025641025643,"measurements":[{"observationId":"vals-medqa:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":50.9,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.5641025641025643,"participants":40,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 4.802. Standardfehler: 0.456. Kosten je Aufgabe in USD: 0.000326. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Maximum output tokens: 16384. Latency in seconds: 4.802. Standard error: 0.456. Cost per task in USD: 0.000326. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":1.4705882352941178,"measurements":[{"observationId":"vals-medscribe:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":50.593,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.4705882352941178,"participants":69,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 11.323. Standardfehler: 1.901. Kosten je Aufgabe in USD: 0.0017. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 11.323. Standard error: 1.901. Cost per task in USD: 0.0017. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":31.03448275862069,"measurements":[{"observationId":"vals-mgsm:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":87.964,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.03448275862069,"participants":30,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 3.499. Standardfehler: 1.993. Kosten je Aufgabe in USD: 0.000147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Maximum output tokens: 16384. Latency in seconds: 3.499. Standard error: 1.993. Cost per task in USD: 0.000147. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":6.4935064935064934,"measurements":[{"observationId":"vals-mmlu-pro:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":69.632,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.4935064935064934,"participants":78,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 5.342. Standardfehler: 0.445. Kosten je Aufgabe in USD: 0.000507. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 5.342. Standard error: 0.445. Cost per task in USD: 0.000507. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":13.461538461538462,"measurements":[{"observationId":"vals-mortgage-tax:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":57.75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.461538461538462,"participants":53,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 2.426. Standardfehler: 0.979. Kosten je Aufgabe in USD: 0.000572. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 2.426. Standard error: 0.979. Cost per task in USD: 0.000572. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":12.068965517241379,"measurements":[{"observationId":"vals-sage:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":34.834,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.068965517241379,"participants":59,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 32.862. Standardfehler: 3.029. Kosten je Aufgabe in USD: 0.003634. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 32.862. Standard error: 3.029. Cost per task in USD: 0.003634. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-tax-eval-v2:overall:together/meta-llama/Llama-4-Scout-17B-16E-Instruct","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":55.192,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":77,"methodology":{"de":"Kennung des Quellmodells: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 7.042. Standardfehler: 0.975. Kosten je Aufgabe in USD: 0.000378. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/meta-llama/Llama-4-Scout-17B-16E-Instruct. Source model name: Llama 4 Scout. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Maximum output tokens: 16384. Latency in seconds: 7.042. Standard error: 0.975. Cost per task in USD: 0.000378. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 16384","en":"Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":11.11111111111111,"measurements":[{"observationId":"curated:scale-visual-toolbench-llama-4-scout","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":1.58,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":11.11111111111111,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/llama-4-scout. Name des Quellmodells: Llama 4 Scout. Anbieter laut Quelle: Meta. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 1,58 % ± 0,04. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-7822fbb7c5cc. Prüfsumme der Quelldaten: 7822fbb7c5ccfaf05dab72443f6fe33aacc3c4831ec7684f25300ddcc9159ea1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/llama-4-scout. Source model name: Llama 4 Scout. Provider reported by source: Meta. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 1.58% ± 0.04. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-7822fbb7c5cc. Source payload hash: 7822fbb7c5ccfaf05dab72443f6fe33aacc3c4831ec7684f25300ddcc9159ea1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":24,"totalFamilies":71,"sourceCount":3,"coverage":0.33741496598639464,"lowerBound":2.9651042806727315,"upperBound":69.22360768203328,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":1.0223313272093764,"upperBound":91.02233132720936,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":1.5954520447460114,"upperBound":58.73830918760315,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":9.843260188087775,"upperBound":69.84326018808778,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":7.515045929680077,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":5.010030619786718,"upperBound":38.343363953120054,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":0.508130081300813,"upperBound":75.50813008130082,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":5.217391304347826,"upperBound":85.21739130434783,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":3.0813780813780816,"upperBound":60.22423522423522,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":3.2051282051282053,"upperBound":78.2051282051282,"provisional":true,"configurationsDiffer":true},"health":{"score":1.3448969331322274,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":1.3448969331322274,"upperBound":1.3448969331322274,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":9.297956771361026,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":4.648978385680513,"upperBound":54.648978385680515,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":6.0344827586206895,"upperBound":56.03448275862069,"provisional":true,"configurationsDiffer":true}}},{"modelId":"longcat-2.0","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":739.75,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":65,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 65,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 65.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":4.533913157029403,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 4,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 4.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":6.4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 6,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 6.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":35.36585365853659,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":869.76,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.36585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-23.416666666666668,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":33.688600556070405,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 33,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 33.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:5c3bdc0d-abab-4526-8079-34de4089bb4a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Name des Quellmodells: LongCat 2.0. Anbieter laut Quelle: LongCat. Modellstand: LongCat 2.0. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/5c3bdc0d-abab-4526-8079-34de4089bb4a/longcat-2-0/longcat-2-0. Source model name: LongCat 2.0. Provider reported by source: LongCat. Model snapshot: LongCat 2.0. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: LongCat 2.0 (Release longcat-2-0, Zeile longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: LongCat 2.0 (release longcat-2-0, row longcat-2-0, UUID 5c3bdc0d-abab-4526-8079-34de4089bb4a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":2.1721005475360884,"upperBound":95.21631823461091,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":11.32404181184669,"upperBound":75.60975609756099,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.75609756097561,"upperBound":76.42276422764228,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":8.739837398373984,"upperBound":83.73983739837398,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mai-code-1-1-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mai-thinking-1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-code-llama-70b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-dbrx","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-deepseek-coder-v2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-deepseek-llm-67b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-falcon-180b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-falcon-3-10b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-falcon-3-7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-granite-code-34b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-internlm2-chat-20b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-internlm3-8b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-kimi-k2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-llama-3.1-405b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-llama-3.1-70b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-llama-3.1-8b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-mistral-7b-v0.3","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-mixtral-8x7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-olmo-2-13b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-olmo-2-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-phi-3-medium","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-qwen2-72b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-qwen2.5-32b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-qwen2.5-72b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-solar-10.7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-starcoder2-15b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-wizardlm-2-8x22b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-yi-1.5-34b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"manual-yi-34b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mercury-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mercury-2-5","contributions":[{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.3333333333333333,"score":2.4613899613899615,"measurements":[{"observationId":"vals-terminal-bench-2-1:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":34.457,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.5714285714285716,"participants":57,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 688.923. Standardfehler: 0.749. Kosten je Aufgabe in USD: 0.214987. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 688.923. Standard error: 0.749. Cost per task in USD: 0.214987. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3513513513513513,"participants":38,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 913.643. Standardfehler: 0. Kosten je Aufgabe in USD: 0.368017. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 913.643. Standard error: 0. Cost per task in USD: 0.368017. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":4.838709677419355,"measurements":[{"observationId":"vals-terminal-bench-science:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.838709677419355,"participants":32,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1693.946. Standardfehler: 0. Kosten je Aufgabe in USD: 0.17804. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 1693.946. Standard error: 0. Cost per task in USD: 0.17804. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":4.838709677419355,"measurements":[{"observationId":"vals-code-migration:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":4.517,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.838709677419355,"participants":63,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 974.083. Standardfehler: 1.048. Kosten je Aufgabe in USD: 0.627223. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 974.083. Standard error: 1.048. Cost per task in USD: 0.627223. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":21.21212121212121,"measurements":[{"observationId":"vals-cyberbench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":45.476,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.21212121212121,"participants":34,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1139.168. Standardfehler: 6.056. Kosten je Aufgabe in USD: 0.12462. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 1139.168. Standard error: 6.056. Cost per task in USD: 0.12462. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":1.694915254237288,"measurements":[{"observationId":"vals-emb:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":8.789,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.694915254237288,"participants":60,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 300.757. Standardfehler: 1.275. Kosten je Aufgabe in USD: 0.417608. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 300.757. Standard error: 1.275. Cost per task in USD: 0.417608. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":1.6666666666666667,"measurements":[{"observationId":"vals-finance-agent-v2:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":18.894,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.6666666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 92.482. Standardfehler: 0.216. Kosten je Aufgabe in USD: 0.106441. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 92.482. Standard error: 0.216. Cost per task in USD: 0.106441. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"vals-ioi:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":2.444,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":34,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 242.94. Standardfehler: 0.683. Kosten je Aufgabe in USD: 0.160874. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 242.94. Standard error: 0.683. Cost per task in USD: 0.160874. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 131.54. Standardfehler: 0. Kosten je Aufgabe in USD: 0.192614. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 131.54. Standard error: 0. Cost per task in USD: 0.192614. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":48.717948717948715,"measurements":[{"observationId":"vals-legal-bench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.06,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.717948717948715,"participants":79,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 3.908. Standardfehler: 0.482. Kosten je Aufgabe in USD: 0.000787. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 3.908. Standard error: 0.482. Cost per task in USD: 0.000787. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":5,"measurements":[{"observationId":"vals-legal-research:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":4.327,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5,"participants":61,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 47.609. Standardfehler: 1.414. Kosten je Aufgabe in USD: 0.058112. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 47.609. Standard error: 1.414. Cost per task in USD: 0.058112. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":5.970149253731344,"measurements":[{"observationId":"vals-medcode:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":31.326,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.970149253731344,"participants":68,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 7.93. Standardfehler: 1.953. Kosten je Aufgabe in USD: 0.004535. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 7.93. Standard error: 1.953. Cost per task in USD: 0.004535. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":2.9411764705882355,"measurements":[{"observationId":"vals-medscribe:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":55.093,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.9411764705882355,"participants":69,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 9.992. Standardfehler: 2.095. Kosten je Aufgabe in USD: 0.004476. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536. Latency in seconds: 9.992. Standard error: 2.095. Cost per task in USD: 0.004476. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Temperature: 1. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":5.128205128205129,"measurements":[{"observationId":"vals-proof-bench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":3,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.128205128205129,"participants":40,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 201.725. Standardfehler: 1.714. Kosten je Aufgabe in USD: 0.044105. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 201.725. Standard error: 1.714. Cost per task in USD: 0.044105. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"vals-skillsbench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":18.061,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":30,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 49.364. Standardfehler: 3.079. Kosten je Aufgabe in USD: 0.101286. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 49.364. Standard error: 3.079. Cost per task in USD: 0.101286. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":0,"measurements":[{"observationId":"vals-snap:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":45.264,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":39,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 68.325. Standardfehler: 1.295. Kosten je Aufgabe in USD: 0.067576. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 68.325. Standard error: 1.295. Cost per task in USD: 0.067576. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-tax-agent-bench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":2.487,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":56,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 29.67. Standardfehler: 1.018. Kosten je Aufgabe in USD: 0.023633. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 29.67. Standard error: 1.018. Cost per task in USD: 0.023633. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":9.722222222222221,"measurements":[{"observationId":"vals-vibe-code-bench:overall:inception/mercury-2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":3.683,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.722222222222221,"participants":73,"methodology":{"de":"Kennung des Quellmodells: inception/mercury-2.5. Name des Quellmodells: Mercury 2.5. Anbieter laut Quelle: Inception. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 193.041. Standardfehler: 1.407. Kosten je Aufgabe in USD: 0.378016. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: inception/mercury-2.5. Source model name: Mercury 2.5. Provider reported by source: Inception. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536. Latency in seconds: 193.041. Standard error: 1.407. Cost per task in USD: 0.378016. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: high. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":17,"totalFamilies":71,"sourceCount":1,"coverage":0.27743764172335605,"lowerBound":3.0691834579195474,"upperBound":75.32541928558395,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":1,"coverage":0.25555555555555565,"lowerBound":1.1867166705876386,"upperBound":75.63116111503207,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.0256410256410258,"upperBound":81.02564102564102,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":0,"upperBound":91.66666666666666,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":0.48022598870056493,"upperBound":57.623083131557706,"provisional":true,"configurationsDiffer":true},"legal":{"score":21.457895474288915,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":16.093421605716685,"upperBound":41.093421605716685,"provisional":true,"configurationsDiffer":true},"health":{"score":4.45566286215979,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":2.9704419081065265,"upperBound":36.303775241439865,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":21.21212121212121,"upperBound":21.21212121212121,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":0,"upperBound":50,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mercury-voice","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-llama-3.2-11b-vision-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-llama-3.3-70b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-llama-4-maverick-17b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-llama-4-scout-17b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-muse-glimmer-30b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"meta-muse-spark-1.3","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":39.473684210526315,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":37.5,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.473684210526315,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 37,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 37.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1581.34,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":88.41463414634147,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":83,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.41463414634147,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":79.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":57.853437865386134,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":79.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 57,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 57.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":40,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":40.6458333333333,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 40,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 40.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":26.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 26,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 26.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1680.99,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":25.714285714285715,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":33.239171374764595,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.714285714285715,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 33,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 33.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":82.92682926829268,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":25,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.92682926829268,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":91.46341463414635,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":48.7025023169602,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":91.46341463414635,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 48,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 48.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":77.77777777777777,"measurements":[{"observationId":"lmarena:webdev:overall:muse-spark-1.3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1658.539986663554,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":77.77777777777777,"participants":37,"methodology":{"de":"Arena-Bewertung aus 8.683 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/muse-spark-1.3-max. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: muse-spark-1.3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 8,683 published pairwise votes.. Source model ID: lmarena/muse-spark-1.3-max. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: muse-spark-1.3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 8.683 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 8,683 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 8.683 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 8,683 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":86.11111111111111,"measurements":[{"observationId":"lmarena:overall:muse-spark-1.3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1494.2128370166465,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":86.11111111111111,"participants":37,"methodology":{"de":"Arena-Bewertung aus 16.731 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/muse-spark-1.3-max. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: muse-spark-1.3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 16,731 published pairwise votes.. Source model ID: lmarena/muse-spark-1.3-max. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: muse-spark-1.3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 16.731 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 16,731 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 16.731 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 16,731 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":86.66666666666667,"measurements":[{"observationId":"lmarena:vision_style_control:overall:muse-spark-1.3-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1289.716856872445,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":86.66666666666667,"participants":31,"methodology":{"de":"Arena-Bewertung aus 4.550 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/muse-spark-1.3-max. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: muse-spark-1.3-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 4,550 published pairwise votes.. Source model ID: lmarena/muse-spark-1.3-max. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: muse-spark-1.3-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 4.550 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 4,550 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 4.550 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control. Reasoning-Stufe: max","en":"Arena rating from 4,550 published pairwise votes.. Harness: Arena Vision, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":80.35714285714286,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:d5170215-69be-4129-849b-26d8d8825bfc","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":82.0231213872832,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.35714285714286,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Xhigh) (Release muse-spark-1-3, Zeile muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/d5170215-69be-4129-849b-26d8d8825bfc/muse-spark-1-3/muse-spark-1-3-xhigh. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 82,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Xhigh) (release muse-spark-1-3, row muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/d5170215-69be-4129-849b-26d8d8825bfc/muse-spark-1-3/muse-spark-1-3-xhigh. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 82.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Xhigh) (Release muse-spark-1-3, Zeile muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Muse Spark 1.3 (Xhigh) (release muse-spark-1-3, row muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Xhigh) (Release muse-spark-1-3, Zeile muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Muse Spark 1.3 (Xhigh) (release muse-spark-1-3, row muse-spark-1-3-xhigh, UUID d5170215-69be-4129-849b-26d8d8825bfc, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":69.58635464733025,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":33.3333333333333,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.58536585365853,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 33,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 33.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":72.285,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":57,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 857.636. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.663249. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 857.636. Standard error: 0.375. Cost per task in USD: 0.663249. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":10.606,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.675675675675677,"participants":38,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 7112.43. Standardfehler: 1.75. Kosten je Aufgabe in USD: 7.036467. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 7112.43. Standard error: 1.75. Cost per task in USD: 7.036467. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":44.14446002805049,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:9999672c-b687-4026-9c15-5cef02ff53bb","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":10.952380952380999,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.869565217391305,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Modellstand: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 11,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/9999672c-b687-4026-9c15-5cef02ff53bb/muse-spark-1-3/muse-spark-1-3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Model snapshot: Muse Spark 1.3 (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 11.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Spark 1.3 (Max) (Release muse-spark-1-3, Zeile muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Spark 1.3 (Max) (release muse-spark-1-3, row muse-spark-1-3, UUID 9999672c-b687-4026-9c15-5cef02ff53bb, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":4.286,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.419354838709676,"participants":32,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4050.414. Standardfehler: 2.438. Kosten je Aufgabe in USD: 4.407744. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4050.414. Standard error: 2.438. Cost per task in USD: 4.407744. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":48.38709677419355,"measurements":[{"observationId":"vals-code-migration:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":27.58,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.38709677419355,"participants":63,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4061.108. Standardfehler: 4.067. Kosten je Aufgabe in USD: 4.616854. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4061.108. Standard error: 4.067. Cost per task in USD: 4.616854. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":57.57575757575758,"measurements":[{"observationId":"vals-cyberbench:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":69.405,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.57575757575758,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1492.536. Standardfehler: 5.758. Kosten je Aufgabe in USD: 4.258407. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1492.536. Standard error: 5.758. Cost per task in USD: 4.258407. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":57.6271186440678,"measurements":[{"observationId":"vals-emb:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":62.712,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.6271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1374.999. Standardfehler: 2.956. Kosten je Aufgabe in USD: 3.552349. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1374.999. Standard error: 2.956. Cost per task in USD: 3.552349. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":95,"measurements":[{"observationId":"vals-finance-agent-v2:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":58.901,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":95,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 341.936. Standardfehler: 0.444. Kosten je Aufgabe in USD: 0.741132. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 341.936. Standard error: 0.444. Cost per task in USD: 0.741132. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":21.21212121212121,"measurements":[{"observationId":"vals-ioi:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":43.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.21212121212121,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1791.133. Standardfehler: 1.448. Kosten je Aufgabe in USD: 2.360741. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1791.133. Standard error: 1.448. Cost per task in USD: 2.360741. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":100,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":22.917,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":62,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1024.261. Standardfehler: 3.416. Kosten je Aufgabe in USD: 2.75654. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1024.261. Standard error: 3.416. Cost per task in USD: 2.75654. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":68.33333333333333,"measurements":[{"observationId":"vals-legal-research:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":40.865,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 530.813. Standardfehler: 3.417. Kosten je Aufgabe in USD: 0.650063. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 530.813. Standard error: 3.417. Cost per task in USD: 0.650063. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4112.719. Standardfehler: 0.5. Kosten je Aufgabe in USD: 11.555338. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4112.719. Standard error: 0.5. Cost per task in USD: 11.555338. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":51.282051282051285,"measurements":[{"observationId":"vals-proof-bench:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.282051282051285,"participants":40,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 534.06. Standardfehler: 5. Kosten je Aufgabe in USD: 0.433712. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 534.06. Standard error: 5. Cost per task in USD: 0.433712. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":85.45454545454545,"measurements":[{"observationId":"vals-tax-agent-bench:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":35.974,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":85.45454545454545,"participants":56,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 505.365. Standardfehler: 3.42. Kosten je Aufgabe in USD: 0.319558. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 505.365. Standard error: 3.42. Cost per task in USD: 0.319558. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":80.55555555555556,"measurements":[{"observationId":"vals-vibe-code-bench:overall:meta/muse_spark_1_3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":82.859,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.55555555555556,"participants":73,"methodology":{"de":"Kennung des Quellmodells: meta/muse_spark_1_3. Name des Quellmodells: Muse Spark 1.3. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 773.797. Standardfehler: 2.901. Kosten je Aufgabe in USD: 2.101762. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse_spark_1_3. Source model name: Muse Spark 1.3. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 773.797. Standard error: 2.901. Cost per task in USD: 2.101762. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":27,"totalFamilies":71,"sourceCount":3,"coverage":0.30949546485260776,"lowerBound":22.111148241034847,"upperBound":91.16160175577406,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":54.92804959827151,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.5222222222222225,"lowerBound":28.684648123541802,"upperBound":76.46242590131955,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":32.57839721254356,"upperBound":96.86411149825784,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":10.256410256410257,"upperBound":90.25641025641026,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":27.642276422764226,"upperBound":94.3089430894309,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":27.790054404303444,"upperBound":86.12338773763678,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":17.222222222222225,"upperBound":97.22222222222223,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":34.01166629980189,"upperBound":91.15452344265903,"provisional":true,"configurationsDiffer":true},"legal":{"score":84.16666666666666,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":42.08333333333333,"upperBound":92.08333333333333,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":57.57575757575757,"upperBound":57.57575757575757,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":31.711309523809526,"upperBound":94.21130952380952,"provisional":true,"configurationsDiffer":true}}},{"modelId":"meta-muse-wonder-1.0","contributions":[{"familyId":"aime","area":"math","label":{"de":"AIME","en":"AIME"},"weight":0.014285714285714285,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"research:meta-muse-wonder-1.0:2","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-otis-mock-aime-2024-2025","benchmarkVersion":"1.1.6","comparisonCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","retrievedAt":"2026-10-19","value":0.9944444444444445,"unit":"ratio","metric":"model_extracted_exact_match","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":77.27272727272727,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: muse-wonder-1.0_max. Epoch-Modellbezeichnung: Muse Wonder. Anzeigenamenfeld: Muse Wonder (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: OTIS Mock AIME 2024-2025. Task-Version: 1.1.6. Lauf-ID: AFigcTGzpgZgMi6j2fpboa. Startzeit: 2026-10-12T19:00:20.000Z. Ursprüngliche Task-Bezeichnung: OTIS Mock AIME 2024-2025. Die CSV-Zeile meldet mean_score=0.9944444444444445 und stderr=0.003883473863441843. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: muse-wonder-1.0_max; Epoch model label: Muse Wonder; Display label field: Muse Wonder (max); Reasoning setting encoded in the alias: max. Task ID: OTIS Mock AIME 2024-2025. Task version: 1.1.6. Run ID: AFigcTGzpgZgMi6j2fpboa. Started at 2026-10-12T19:00:20.000Z. Original task label: OTIS Mock AIME 2024-2025. The published row reports mean_score=0.9944444444444445 and stderr=0.003883473863441843. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"OTIS Mock AIME 2024-2025\",\"OTIS Mock AIME 2024-2025\",\"1.1.6\",\"model_extracted_exact_match\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":97.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1658.65,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":72.5609756097561,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":80.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":72.5609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 80,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 80.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":68.09455131207758,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 68,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 68.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":30.599999999999998,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 30,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 30.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":98.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1770.34,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":25.483333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"frontiermath","area":"math","label":{"de":"FrontierMath","en":"FrontierMath"},"weight":0.014285714285714285,"coverage":0.5,"score":68.18181818181819,"measurements":[{"observationId":"research:meta-muse-wonder-1.0:0","sourceId":"epoch-ai","sourceName":"Epoch AI benchmark run dataset snapshot","sourceUrl":"https://epoch.ai/data/benchmarks.csv","benchmarkId":"epoch-frontiermath-tiers-1-3-v2-private","benchmarkVersion":"2.0.0","comparisonCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","retrievedAt":"2026-10-19","value":0.8210526315789474,"unit":"ratio","metric":"verification_code","task":"overall","evaluator":"Epoch AI benchmark run dataset snapshot","higherIsBetter":true,"percentile":68.18181818181819,"participants":23,"methodology":{"de":"Unabhängige Evaluation. Epoch-Laufalias: muse-wonder-1.0_max. Epoch-Modellbezeichnung: Muse Wonder. Anzeigenamenfeld: Muse Wonder (max). Im Alias kodierte Reasoning-Stufe: max. Task-ID: FrontierMath-Tiers-1-3-v2-Private. Task-Version: 2.0.0. Lauf-ID: hwVgELmPofPY3Vqm7BRLxs. Startzeit: 2026-10-12T19:00:20.000Z. Ursprüngliche Task-Bezeichnung: FrontierMath-Tiers-1-3-v2-Private. Die CSV-Zeile meldet mean_score=0.8210526315789474 und stderr=0.02274515950332741. Die CSV enthält keine Spalte zur Stichprobengröße. Sie wird daher nicht abgeleitet. Der Standardfehler wird nicht als Konfidenzintervall ausgegeben.\n\nMeta-Score-Auswahl: Pro Modell wird innerhalb desselben Epoch-Quell-Testdesigns, Original-Tasks, derselben Version und desselben Scorers die höchste dokumentierte Reasoning-Stufe verwendet, unabhängig vom Messwert. Diese abgeleitete Score-Kopie behauptet kein vergleichbares Rechenbudget; Epoch veröffentlicht im Capture weder ein einheitliches Budget noch alle Inferenzparameter. Die Rohbeobachtung im Modellprofil bleibt unverändert. Verwendete Reasoning-Stufe: max.","en":"Independent evaluation. Epoch run alias: muse-wonder-1.0_max; Epoch model label: Muse Wonder; Display label field: Muse Wonder (max); Reasoning setting encoded in the alias: max. Task ID: FrontierMath-Tiers-1-3-v2-Private. Task version: 2.0.0. Run ID: hwVgELmPofPY3Vqm7BRLxs. Started at 2026-10-12T19:00:20.000Z. Original task label: FrontierMath-Tiers-1-3-v2-Private. The published row reports mean_score=0.8210526315789474 and stderr=0.02274515950332741. The CSV has no sample-count column, so none is inferred. Standard error is not presented as a confidence interval.\n\nMeta-score selection: Per model, the highest documented reasoning tier is selected within the same Epoch source test design, original task, version, and scorer, independently of the measured value. This derived score copy does not claim matched compute budgets; the capture publishes neither a common budget nor all inference parameters. The raw model-profile observation remains unchanged. Selected reasoning tier: max."},"comparisonMethodology":{"de":"Veröffentlichte Reasoning-Stufe: max. Weitere Inferenzparameter und ein Rechenbudget nennt der Capture nicht.","en":"Published reasoning tier: max. The capture does not report other inference parameters or a compute budget."},"scoreComparisonSetup":{"publicationCohort":"epoch-score-design-v1:[\"FrontierMath-Tiers-1-3-v2-Private\",\"FrontierMath-Tiers-1-3-v2-Private\",\"2.0.0\",\"verification_code\"]","datasetType":"not-reported-in-captured-run-csv","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":50.0926784059314,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 50,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 50.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":72.22222222222223,"measurements":[{"observationId":"lmarena:webdev:overall:muse-wonder-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1650.947320232814,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":72.22222222222223,"participants":37,"methodology":{"de":"Arena-Bewertung aus 1.228 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/muse-wonder-max. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: muse-wonder-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 1,228 published pairwise votes.. Source model ID: lmarena/muse-wonder-max. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: muse-wonder-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 1.228 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 1,228 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 1.228 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev. Reasoning-Stufe: max","en":"Arena rating from 1,228 published pairwise votes.. Harness: Arena Code Arena WebDev. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"lmarena:overall:muse-wonder-max","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1512.6713222356636,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":100,"participants":37,"methodology":{"de":"Arena-Bewertung aus 1.944 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/muse-wonder-max. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: muse-wonder-max. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Reasoning-Stufe: max. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 1,944 published pairwise votes.. Source model ID: lmarena/muse-wonder-max. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: muse-wonder-max. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Reasoning effort: max. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 1.944 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 1,944 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 1.944 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control. Reasoning-Stufe: max","en":"Arena rating from 1,944 published pairwise votes.. Harness: Arena Text, style control. Reasoning effort: max"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":92.52636783124589,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:ff2f3f3d-783f-4e27-8a39-47237501a313","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":52.020202020202,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 52,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/ff2f3f3d-783f-4e27-8a39-47237501a313/muse-wonder/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Model snapshot: Muse Wonder (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 52.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder (Max) (Release muse-wonder, Zeile muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder (Max) (release muse-wonder, row muse-wonder, UUID ff2f3f3d-783f-4e27-8a39-47237501a313, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":49.495,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.08108108108108,"participants":38,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3280.687. Standardfehler: 2.201. Kosten je Aufgabe in USD: 14.657694. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 3280.687. Standard error: 2.201. Cost per task in USD: 14.657694. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":74.19354838709677,"measurements":[{"observationId":"vals-terminal-bench-science:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":22.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.19354838709677,"participants":32,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 5584.198. Standardfehler: 5.055. Kosten je Aufgabe in USD: 19.609644. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 5584.198. Standard error: 5.055. Cost per task in USD: 19.609644. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":28.94736842105263,"measurements":[{"observationId":"vals-biomysterybench:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":67.037,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.94736842105263,"participants":20,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1081.905. Standardfehler: 0.37. Kosten je Aufgabe in USD: 3.228928. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1081.905. Standard error: 0.37. Cost per task in USD: 3.228928. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":93.54838709677419,"measurements":[{"observationId":"vals-code-migration:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":62.552,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.54838709677419,"participants":63,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 10267.754. Standardfehler: 4.354. Kosten je Aufgabe in USD: 69.535349. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 10267.754. Standard error: 4.354. Cost per task in USD: 69.535349. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":100,"measurements":[{"observationId":"vals-cyberbench:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":80.357,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 824.81. Standardfehler: 5.133. Kosten je Aufgabe in USD: 3.658792. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 824.81. Standard error: 5.133. Cost per task in USD: 3.658792. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":74.57627118644068,"measurements":[{"observationId":"vals-emb:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":67.084,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.57627118644068,"participants":60,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1034.437. Standardfehler: 2.891. Kosten je Aufgabe in USD: 5.599337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1034.437. Standard error: 2.891. Cost per task in USD: 5.599337. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":56.637,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 368.853. Standardfehler: 0.382. Kosten je Aufgabe in USD: 2.668231. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 368.853. Standard error: 0.382. Cost per task in USD: 2.668231. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":60.60606060606061,"measurements":[{"observationId":"vals-ioi:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":66.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.60606060606061,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1502.067. Standardfehler: 4.908. Kosten je Aufgabe in USD: 4.541594. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1502.067. Standard error: 4.908. Cost per task in USD: 4.541594. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":96.72131147540983,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":17.083,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.72131147540983,"participants":62,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1370.229. Standardfehler: 3.103. Kosten je Aufgabe in USD: 7.081987. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1370.229. Standard error: 3.103. Cost per task in USD: 7.081987. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":94.16666666666667,"measurements":[{"observationId":"vals-legal-research:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":50.481,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.16666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 927.111. Standardfehler: 3.475. Kosten je Aufgabe in USD: 3.980242. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 927.111. Standard error: 3.475. Cost per task in USD: 3.980242. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":40,"measurements":[{"observationId":"vals-mysterymechanism:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":27.477,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40,"participants":21,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1096.935. Standardfehler: 3.003. Kosten je Aufgabe in USD: 3.389451. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1096.935. Standard error: 3.003. Cost per task in USD: 3.389451. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":74.35897435897436,"measurements":[{"observationId":"vals-proof-bench:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":84,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":74.35897435897436,"participants":40,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 929.053. Standardfehler: 3.685. Kosten je Aufgabe in USD: 1.313345. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 929.053. Standard error: 3.685. Cost per task in USD: 1.313345. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":86.84210526315789,"measurements":[{"observationId":"vals-snap:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":70.23,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.84210526315789,"participants":39,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1184.49. Standardfehler: 1.189. Kosten je Aufgabe in USD: 2.457181. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1184.49. Standard error: 1.189. Cost per task in USD: 2.457181. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":61.53846153846154,"measurements":[{"observationId":"vals-swebench-multimodal:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":45.417,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.53846153846154,"participants":14,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 581.998. Standardfehler: 2.275. Kosten je Aufgabe in USD: 1.693517. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 581.998. Standard error: 2.275. Cost per task in USD: 1.693517. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":87.27272727272727,"measurements":[{"observationId":"vals-tax-agent-bench:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":36.269,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.27272727272727,"participants":56,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 492.843. Standardfehler: 3.47. Kosten je Aufgabe in USD: 1.884439. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 492.843. Standard error: 3.47. Cost per task in USD: 1.884439. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":76.19047619047619,"measurements":[{"observationId":"vals-vcb-1-100:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":26.55,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.19047619047619,"participants":22,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1868.026. Standardfehler: 4.333. Kosten je Aufgabe in USD: 13.721565. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1868.026. Standard error: 4.333. Cost per task in USD: 13.721565. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":81.94444444444444,"measurements":[{"observationId":"vals-vibe-code-bench:overall:meta/muse-wonder","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":83.089,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.94444444444444,"participants":73,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder. Name des Quellmodells: Muse Wonder. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 350.076. Standardfehler: 2.49. Kosten je Aufgabe in USD: 1.802723. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder. Source model name: Muse Wonder. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 350.076. Standard error: 2.49. Cost per task in USD: 1.802723. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":28,"totalFamilies":71,"sourceCount":4,"coverage":0.3533446712018141,"lowerBound":29.793351944144995,"upperBound":94.45888482396359,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":3,"coverage":0.47777777777777797,"lowerBound":36.3220714341879,"upperBound":88.5442936564101,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":30.836236933797913,"upperBound":95.1219512195122,"provisional":true,"configurationsDiffer":true},"math":{"score":74.28904428904428,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.5,"lowerBound":37.14452214452214,"upperBound":87.14452214452214,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":28.455284552845526,"upperBound":95.1219512195122,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":24.390243902439025,"upperBound":99.39024390243902,"provisional":true,"configurationsDiffer":true},"general":{"score":56.315789473684205,"coveredFamilies":3,"totalFamilies":5,"sourceCount":2,"coverage":0.6000000000000001,"lowerBound":33.78947368421053,"upperBound":73.78947368421052,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":35.02604739892875,"upperBound":92.16890454178589,"provisional":true,"configurationsDiffer":true},"legal":{"score":95.44398907103826,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":47.72199453551913,"upperBound":97.72199453551913,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":100,"upperBound":100,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":43.421052631578945,"upperBound":93.42105263157895,"provisional":true,"configurationsDiffer":true}}},{"modelId":"meta-muse-wonder-full-1.0","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1729.32,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":83.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 83,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 83.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":71.33074973223975,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 71,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 71.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":32.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 32,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 32.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":1857.09,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":92.6829268292683,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":30.15,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":92.6829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":96.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":53.6144578313253,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 53,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 53.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":96.72050098879367,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Max","retrievedAt":"2026-10-19","value":57.07070707070711,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":98.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Modellstand: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Max. Angezeigter Wert: 57,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Source model ID: artificial-analysis/e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2/muse-wonder-full/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Model snapshot: Muse Wonder Full (Max). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Max. Score display: 57.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Muse Wonder Full (Max) (Release muse-wonder-full, Zeile muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Max","en":"Exact AA configuration: Muse Wonder Full (Max) (release muse-wonder-full, row muse-wonder-full, UUID e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Max). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Max"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Max","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":58.081,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":90.54054054054055,"participants":38,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4330.665. Standardfehler: 3.312. Kosten je Aufgabe in USD: 10.528007. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4330.665. Standard error: 3.312. Cost per task in USD: 10.528007. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":83.87096774193549,"measurements":[{"observationId":"vals-terminal-bench-science:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":37.143,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.87096774193549,"participants":32,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 12919.209. Standardfehler: 5.817. Kosten je Aufgabe in USD: 22.814086. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 12919.209. Standard error: 5.817. Cost per task in USD: 22.814086. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Mini-SWE-agent. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":68.42105263157895,"measurements":[{"observationId":"vals-biomysterybench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":73.333,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":68.42105263157895,"participants":20,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1574.165. Standardfehler: 0.642. Kosten je Aufgabe in USD: 3.473154. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1574.165. Standard error: 0.642. Cost per task in USD: 3.473154. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: Terminus 2. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":91.93548387096774,"measurements":[{"observationId":"vals-code-migration:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":61.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.93548387096774,"participants":63,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 13956.531. Standardfehler: 4.419. Kosten je Aufgabe in USD: 38.410556. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 13956.531. Standard error: 4.419. Cost per task in USD: 38.410556. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"vals-cyberbench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":73.631,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.27272727272727,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2126.512. Standardfehler: 5.564. Kosten je Aufgabe in USD: 3.627348. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 2126.512. Standard error: 5.564. Cost per task in USD: 3.627348. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":88.13559322033899,"measurements":[{"observationId":"vals-emb:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":71.917,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.13559322033899,"participants":60,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1684.504. Standardfehler: 2.658. Kosten je Aufgabe in USD: 4.58223. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1684.504. Standard error: 2.658. Cost per task in USD: 4.58223. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":98.33333333333333,"measurements":[{"observationId":"vals-finance-agent-v2:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":59.061,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 895.804. Standardfehler: 0.215. Kosten je Aufgabe in USD: 2.389758. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 895.804. Standard error: 0.215. Cost per task in USD: 2.389758. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":72.72727272727273,"measurements":[{"observationId":"vals-ioi:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":79.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.72727272727273,"participants":34,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 5446.154. Standardfehler: 2.992. Kosten je Aufgabe in USD: 7.078463. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 5446.154. Standard error: 2.992. Cost per task in USD: 7.078463. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":98.36065573770492,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":21.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":98.36065573770492,"participants":62,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3201.328. Standardfehler: 3.537. Kosten je Aufgabe in USD: 5.413598. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 3201.328. Standard error: 3.537. Cost per task in USD: 5.413598. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":97.43589743589743,"measurements":[{"observationId":"vals-legal-bench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":87.418,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":97.43589743589743,"participants":79,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 38.267. Standardfehler: 0.413. Kosten je Aufgabe in USD: 0.025453. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 38.267. Standard error: 0.413. Cost per task in USD: 0.025453. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":96.66666666666667,"measurements":[{"observationId":"vals-legal-research:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":53.365,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.66666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1654.378. Standardfehler: 3.467. Kosten je Aufgabe in USD: 3.595401. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1654.378. Standard error: 3.467. Cost per task in USD: 3.595401. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":80.59701492537313,"measurements":[{"observationId":"vals-medcode:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":49.932,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.59701492537313,"participants":68,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 182.7. Standardfehler: 2.177. Kosten je Aufgabe in USD: 0.247179. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 182.7. Standard error: 2.177. Cost per task in USD: 0.247179. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":91.17647058823529,"measurements":[{"observationId":"vals-medscribe:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":89.335,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.17647058823529,"participants":69,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 115.718. Standardfehler: 2.004. Kosten je Aufgabe in USD: 0.19181. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 115.718. Standard error: 2.004. Cost per task in USD: 0.19181. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":62.5,"measurements":[{"observationId":"vals-mysterymechanism:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":35.135,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.5,"participants":21,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2494.329. Standardfehler: 3.211. Kosten je Aufgabe in USD: 4.163398. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 2494.329. Standard error: 3.211. Cost per task in USD: 4.163398. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":60.204081632653065,"measurements":[{"observationId":"vals-programbench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0.5,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.204081632653065,"participants":50,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 21822.228. Standardfehler: 0.5. Kosten je Aufgabe in USD: 28.645593. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 21822.228. Standard error: 0.5. Cost per task in USD: 28.645593. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":82.05128205128206,"measurements":[{"observationId":"vals-proof-bench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":94,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.05128205128206,"participants":40,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 695.558. Standardfehler: 2.387. Kosten je Aufgabe in USD: 0.856208. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 695.558. Standard error: 2.387. Cost per task in USD: 0.856208. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":94.73684210526316,"measurements":[{"observationId":"vals-snap:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":73.748,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.73684210526316,"participants":39,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 4364.892. Standardfehler: 1.145. Kosten je Aufgabe in USD: 4.635817. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 4364.892. Standard error: 1.145. Cost per task in USD: 4.635817. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sre-bench","area":"general","label":{"de":"SRE Bench","en":"SRE Bench"},"weight":0.014285714285714285,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"vals-sre-bench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","benchmarkId":"vals-sre-bench","benchmarkVersion":"1","comparisonCohort":"vals-sre-bench:1:overall","retrievedAt":"2026-10-15","value":11.832,"unit":"%","metric":"fully solved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":25,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 20035.497. Standardfehler: 1.999. Kosten je Aufgabe in USD: 33.810651. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-srebench-2026-10-15-a10b2c964f58. Prüfsumme der Quelldaten: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 20035.497. Standard error: 1.999. Cost per task in USD: 33.810651. Source ID: vals-ai. Evidence capture ID: vals-srebench-2026-10-15-a10b2c964f58. Source payload hash: a10b2c964f5878b0fc21c888703f311a5ef7d7a74fc3c529cab3ddd9fa474e2b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-sre-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"vals-swebench-multimodal:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":47.292,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":14,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 862.026. Standardfehler: 2.281. Kosten je Aufgabe in USD: 1.860894. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 862.026. Standard error: 2.281. Cost per task in USD: 1.860894. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":94.54545454545455,"measurements":[{"observationId":"vals-tax-agent-bench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":42.997,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.54545454545455,"participants":56,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1084.296. Standardfehler: 3.572. Kosten je Aufgabe in USD: 2.147235. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 1084.296. Standard error: 3.572. Cost per task in USD: 2.147235. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"vals-vcb-1-100:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":33.978,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":22,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 7008.163. Standardfehler: 5.318. Kosten je Aufgabe in USD: 19.5403. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 7008.163. Standard error: 5.318. Cost per task in USD: 19.5403. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":86.11111111111111,"measurements":[{"observationId":"vals-vibe-code-bench:overall:meta/muse-wonder-full","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":85.996,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.11111111111111,"participants":73,"methodology":{"de":"Kennung des Quellmodells: meta/muse-wonder-full. Name des Quellmodells: Muse Wonder Full. Anbieter laut Quelle: Meta. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 648.336. Standardfehler: 1.933. Kosten je Aufgabe in USD: 2.293448. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: meta/muse-wonder-full. Source model name: Muse Wonder Full. Provider reported by source: Meta. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072. Latency in seconds: 648.336. Standard error: 1.933. Cost per task in USD: 2.293448. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: max. Temperatur: 1. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Reasoning effort: max. Temperature: 1. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"max","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":29,"totalFamilies":71,"sourceCount":2,"coverage":0.3973922902494331,"lowerBound":34.37034336168237,"upperBound":94.63111433673906,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":2,"coverage":0.47777777777777797,"lowerBound":39.62112294190525,"upperBound":91.84334516412744,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":34.58188153310105,"upperBound":98.86759581881535,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":16.410256410256412,"upperBound":96.41025641025641,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":30.894308943089435,"upperBound":97.5609756097561,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":25,"upperBound":100,"provisional":true,"configurationsDiffer":true},"general":{"score":65.8625730994152,"coveredFamilies":3,"totalFamilies":5,"sourceCount":1,"coverage":0.6000000000000001,"lowerBound":39.51754385964913,"upperBound":79.51754385964912,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":40.144911585589554,"upperBound":97.2877687284467,"provisional":true,"configurationsDiffer":true},"legal":{"score":97.48773994675635,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":73.11580496006725,"upperBound":98.11580496006725,"provisional":true,"configurationsDiffer":true},"health":{"score":85.88674275680421,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":57.257828504536135,"upperBound":90.59116183786946,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":77.27272727272727,"upperBound":77.27272727272727,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":47.36842105263158,"upperBound":97.36842105263159,"provisional":true,"configurationsDiffer":true}}},{"modelId":"microsoft-phi-4-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mimo-v2.5","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":753.75,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":33.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":73,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 73,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 73.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":18.42951233004315,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 18,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 18.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 4,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 4.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":43.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":997.19,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":55.79710144927536,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":67.14285714285711,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.79710144927536,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 67,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 67.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-9.833333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":39.473684210526315,"measurements":[{"observationId":"vals-gpqa:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":81.566,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.473684210526315,"participants":77,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 89.393. Standardfehler: 2.054. Kosten je Aufgabe in USD: 0.002538. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 89.393. Standard error: 2.054. Cost per task in USD: 0.002538. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":27.201112140871196,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 27,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 27.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":16.666666666666668,"measurements":[{"observationId":"lmarena:webdev:overall:mimo-v2.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1438.1786125279186,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":16.666666666666668,"participants":37,"methodology":{"de":"Arena-Bewertung aus 14.953 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: mimo-v2.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 14,953 published pairwise votes.. Source model ID: lmarena/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: mimo-v2.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 14.953 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 14,953 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 14.953 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 14,953 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":5.555555555555555,"measurements":[{"observationId":"lmarena:overall:mimo-v2.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1433.803517650687,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":5.555555555555555,"participants":37,"methodology":{"de":"Arena-Bewertung aus 47.388 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: mimo-v2.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 47,388 published pairwise votes.. Source model ID: lmarena/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: mimo-v2.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 47.388 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 47,388 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 47.388 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 47,388 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"lmarena:vision_style_control:overall:mimo-v2.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1233.5965008769429,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":6.666666666666667,"participants":31,"methodology":{"de":"Arena-Bewertung aus 28.794 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: mimo-v2.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 28,794 published pairwise votes.. Source model ID: lmarena/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: mimo-v2.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 28.794 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 28,794 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 28.794 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 28,794 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":39.171732522796354,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":75.4335260115607,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.42857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 75,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 75.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":80,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.914893617021278,"participants":48,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 42.47. Standardfehler: 0.962. Kosten je Aufgabe in USD: 0.001396. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 42.47. Standard error: 0.962. Cost per task in USD: 0.001396. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":28.8135593220339,"measurements":[{"observationId":"vals-swebench:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":71,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.8135593220339,"participants":60,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 244.053. Standardfehler: 2.031. Kosten je Aufgabe in USD: 0.010032. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 244.053. Standard error: 2.031. Cost per task in USD: 0.010032. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":23.976480836236934,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:22d09131-343b-4adf-8760-533e20a2155f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/22d09131-343b-4adf-8760-533e20a2155f/mimo-v2-5-0424/mimo-v2-5-0424. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5 (Release mimo-v2-5-0424, Zeile mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiMo-V2.5 (release mimo-v2-5-0424, row mimo-v2-5-0424, UUID 22d09131-343b-4adf-8760-533e20a2155f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":60.674,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":57,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 775.239. Standardfehler: 5.207. Kosten je Aufgabe in USD: 0.021658. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 775.239. Standard error: 5.207. Cost per task in USD: 0.021658. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":29.032258064516128,"measurements":[{"observationId":"vals-code-migration:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":14.228,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.032258064516128,"participants":63,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2274.632. Standardfehler: 3.687. Kosten je Aufgabe in USD: 0.102802. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2274.632. Standard error: 3.687. Cost per task in USD: 0.102802. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":33.78378378378378,"measurements":[{"observationId":"vals-corp-fin-v2:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":59.907,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":33.78378378378378,"participants":75,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 17.625. Standardfehler: 0.966. Kosten je Aufgabe in USD: 0.011733. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 17.625. Standard error: 0.966. Cost per task in USD: 0.011733. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":35.59322033898305,"measurements":[{"observationId":"vals-emb:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":55.092,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.59322033898305,"participants":60,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1837.949. Standardfehler: 3.127. Kosten je Aufgabe in USD: 0.083304. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1837.949. Standard error: 3.127. Cost per task in USD: 0.083304. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"vals-finance-agent-v2:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":36.728,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.333333333333334,"participants":61,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 339.26. Standardfehler: 0.109. Kosten je Aufgabe in USD: 0.08693. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 339.26. Standard error: 0.109. Cost per task in USD: 0.08693. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":42.857142857142854,"measurements":[{"observationId":"vals-lcb:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.512,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":78,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 108.607. Standardfehler: 1.072. Kosten je Aufgabe in USD: 0.002243. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 108.607. Standard error: 1.072. Cost per task in USD: 0.002243. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":37.704918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.704918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 419.277. Standardfehler: 0. Kosten je Aufgabe in USD: 0.036252. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 419.277. Standard error: 0. Cost per task in USD: 0.036252. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":24.358974358974358,"measurements":[{"observationId":"vals-legal-bench:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":78.886,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.358974358974358,"participants":79,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 11.184. Standardfehler: 0.506. Kosten je Aufgabe in USD: 0.000264. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 11.184. Standard error: 0.506. Cost per task in USD: 0.000264. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":9.166666666666666,"measurements":[{"observationId":"vals-legal-research:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":9.135,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.166666666666666,"participants":61,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 391.469. Standardfehler: 2.002. Kosten je Aufgabe in USD: 0.033318. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 391.469. Standard error: 2.002. Cost per task in USD: 0.033318. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":7.462686567164179,"measurements":[{"observationId":"vals-medcode:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":31.895,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.462686567164179,"participants":68,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 17.704. Standardfehler: 2.025. Kosten je Aufgabe in USD: 0.002162. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 17.704. Standard error: 2.025. Cost per task in USD: 0.002162. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":14.705882352941176,"measurements":[{"observationId":"vals-medscribe:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":72.151,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.705882352941176,"participants":69,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 20.261. Standardfehler: 1.851. Kosten je Aufgabe in USD: 0.001351. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 20.261. Standard error: 1.851. Cost per task in USD: 0.001351. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"vals-mmlu-pro:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":82.931,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":78,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 30.248. Standardfehler: 0.372. Kosten je Aufgabe in USD: 0.000774. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 30.248. Standard error: 0.372. Cost per task in USD: 0.000774. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":17.307692307692307,"measurements":[{"observationId":"vals-mortgage-tax:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":59.26,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.307692307692307,"participants":53,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 32.939. Standardfehler: 0.979. Kosten je Aufgabe in USD: 0.001523. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 32.939. Standard error: 0.979. Cost per task in USD: 0.001523. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":17.94871794871795,"measurements":[{"observationId":"vals-proof-bench:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":16,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.94871794871795,"participants":40,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1255.813. Standardfehler: 3.685. Kosten je Aufgabe in USD: 0.05707. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1255.813. Standard error: 3.685. Cost per task in USD: 0.05707. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":34.48275862068966,"measurements":[{"observationId":"vals-sage:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":43.265,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.48275862068966,"participants":59,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 52.66. Standardfehler: 3.387. Kosten je Aufgabe in USD: 0.004398. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 52.66. Standard error: 3.387. Cost per task in USD: 0.004398. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":16.363636363636363,"measurements":[{"observationId":"vals-tax-agent-bench:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":7.772,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.363636363636363,"participants":56,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 500.826. Standardfehler: 1.779. Kosten je Aufgabe in USD: 0.017023. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 500.826. Standard error: 1.779. Cost per task in USD: 0.017023. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":40.78947368421053,"measurements":[{"observationId":"vals-tax-eval-v2:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.832,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.78947368421053,"participants":77,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 26.3. Standardfehler: 0.881. Kosten je Aufgabe in USD: 0.000729. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 26.3. Standard error: 0.881. Cost per task in USD: 0.000729. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":38.888888888888886,"measurements":[{"observationId":"vals-vibe-code-bench:overall:xiaomi/mimo-v2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":42.174,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.888888888888886,"participants":73,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5. Name des Quellmodells: MiMo-V2.5. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1580.348. Standardfehler: 4.568. Kosten je Aufgabe in USD: 0.068395. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5. Source model name: MiMo-V2.5. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1580.348. Standard error: 4.568. Cost per task in USD: 0.068395. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":32,"totalFamilies":71,"sourceCount":3,"coverage":0.39739229024943323,"lowerBound":11.097381115199429,"upperBound":71.35815209025611,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":3,"coverage":0.34444444444444455,"lowerBound":10.522403779714853,"upperBound":76.07795933527039,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":13.882266642215292,"upperBound":71.02512378507242,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":3.5897435897435894,"upperBound":83.58974358974359,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":41.35254988913526,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":27.56836659275684,"upperBound":60.901699926090174,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":11.178861788617887,"upperBound":86.1788617886179,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":12.270531400966183,"upperBound":72.27053140096618,"provisional":true,"configurationsDiffer":true},"finance":{"score":26.195189968606563,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":22.453019973091337,"upperBound":36.73873425880563,"provisional":true,"configurationsDiffer":true},"legal":{"score":23.74351968614264,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":17.80763976460698,"upperBound":42.807639764606975,"provisional":true,"configurationsDiffer":true},"health":{"score":11.084284460052677,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":7.389522973368452,"upperBound":40.722856306701786,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":22.91919959473151,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":11.459599797365755,"upperBound":61.459599797365755,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":17.24137931034483,"upperBound":67.24137931034483,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mimo-v2.5-pro","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":22.36842105263158,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":20,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":22.36842105263158,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 20,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 20.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":51.21951219512195,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":882.79,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.21951219512195,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":79.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 79,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 79.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":3.8461538461538463,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":2.43362831858407,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.8461538461538463,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 2,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 2.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":13.703616508004313,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 13,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 13.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":4,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 4,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 4.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":1120.77,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":41.42857142857143,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":38.229755178907695,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.42857142857143,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 38,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 38.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":3.25,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":42.10526315789474,"measurements":[{"observationId":"vals-gpqa:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":82.576,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.10526315789474,"participants":77,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 251.332. Standardfehler: 2.468. Kosten je Aufgabe in USD: 0.006654. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 251.332. Standard error: 2.468. Cost per task in USD: 0.006654. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":54.8780487804878,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":35.6811862835959,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.8780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 35,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 35.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":25,"measurements":[{"observationId":"lmarena:webdev:overall:mimo-v2.5-pro","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1480.461230744728,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":25,"participants":37,"methodology":{"de":"Arena-Bewertung aus 21.811 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: mimo-v2.5-pro. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 21,811 published pairwise votes.. Source model ID: lmarena/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: mimo-v2.5-pro. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 21.811 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 21,811 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 21.811 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 21,811 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"lmarena:overall:mimo-v2.5-pro","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1467.4455405771062,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":44.44444444444444,"participants":37,"methodology":{"de":"Arena-Bewertung aus 74.943 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: mimo-v2.5-pro. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 74,943 published pairwise votes.. Source model ID: lmarena/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: mimo-v2.5-pro. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 74.943 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 74,943 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 74.943 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 74,943 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":38.983050847457626,"measurements":[{"observationId":"vals-swebench:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":74,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.983050847457626,"participants":60,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 470.814. Standardfehler: 1.964. Kosten je Aufgabe in USD: 0.022968. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 470.814. Standard error: 1.964. Cost per task in USD: 0.022968. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":19.63967887748376,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:00f1248e-78e3-4230-8dc8-5e13ba8645e2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Modellstand: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/00f1248e-78e3-4230-8dc8-5e13ba8645e2/mimo-v2-5-pro/mimo-v2-5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Model snapshot: MiMo-V2.5-Pro (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiMo-V2.5-Pro (Reasoning) (Release mimo-v2-5-pro, Zeile mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: MiMo-V2.5-Pro (Reasoning) (release mimo-v2-5-pro, row mimo-v2-5-pro, UUID 00f1248e-78e3-4230-8dc8-5e13ba8645e2, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":57.303,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.392857142857146,"participants":57,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 866.253. Standardfehler: 5.273. Kosten je Aufgabe in USD: 0.043613. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 866.253. Standard error: 5.273. Cost per task in USD: 0.043613. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":0.505,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.756756756756757,"participants":38,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 7622.037. Standardfehler: 0.505. Kosten je Aufgabe in USD: 0.23303. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 7622.037. Standard error: 0.505. Cost per task in USD: 0.23303. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":38.70967741935484,"measurements":[{"observationId":"vals-code-migration:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":21.557,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.70967741935484,"participants":63,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2932.892. Standardfehler: 4.127. Kosten je Aufgabe in USD: 0.2383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2932.892. Standard error: 4.127. Cost per task in USD: 0.2383. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":54.054054054054056,"measurements":[{"observationId":"vals-corp-fin-v2:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.422,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.054054054054056,"participants":75,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 18.899. Standardfehler: 0.959. Kosten je Aufgabe in USD: 0.037238. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 18.899. Standard error: 0.959. Cost per task in USD: 0.037238. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":37.28813559322034,"measurements":[{"observationId":"vals-emb:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":55.228,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.28813559322034,"participants":60,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1747.98. Standardfehler: 3.112. Kosten je Aufgabe in USD: 0.221264. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1747.98. Standard error: 3.112. Cost per task in USD: 0.221264. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":23.333333333333332,"measurements":[{"observationId":"vals-finance-agent-v2:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":41.502,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 497.373. Standardfehler: 1.193. Kosten je Aufgabe in USD: 0.206445. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 497.373. Standard error: 1.193. Cost per task in USD: 0.206445. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":38.96103896103896,"measurements":[{"observationId":"vals-lcb:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.354,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.96103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 271.074. Standardfehler: 1.069. Kosten je Aufgabe in USD: 0.006808. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 271.074. Standard error: 1.069. Cost per task in USD: 0.006808. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":43.442622950819676,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":2.083,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.442622950819676,"participants":62,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 544.226. Standardfehler: 0.833. Kosten je Aufgabe in USD: 0.107095. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 544.226. Standard error: 0.833. Cost per task in USD: 0.107095. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":17.94871794871795,"measurements":[{"observationId":"vals-legal-bench:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":77.129,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.94871794871795,"participants":79,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 27.142. Standardfehler: 0.55. Kosten je Aufgabe in USD: 0.000845. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 27.142. Standard error: 0.55. Cost per task in USD: 0.000845. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":23.333333333333332,"measurements":[{"observationId":"vals-legal-research:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":15.865,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.333333333333332,"participants":61,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 416.437. Standardfehler: 2.539. Kosten je Aufgabe in USD: 0.082648. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 416.437. Standard error: 2.539. Cost per task in USD: 0.082648. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":11.940298507462687,"measurements":[{"observationId":"vals-medcode:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":32.484,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.940298507462687,"participants":68,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 38.448. Standardfehler: 1.907. Kosten je Aufgabe in USD: 0.006719. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 38.448. Standard error: 1.907. Cost per task in USD: 0.006719. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":57.35294117647059,"measurements":[{"observationId":"vals-medscribe:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.73,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.35294117647059,"participants":69,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 90.71. Standardfehler: 2.063. Kosten je Aufgabe in USD: 0.00603. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 90.71. Standard error: 2.063. Cost per task in USD: 0.00603. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-mmlu-pro:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":84.595,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":78,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 67.757. Standardfehler: 0.453. Kosten je Aufgabe in USD: 0.002085. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 67.757. Standard error: 0.453. Cost per task in USD: 0.002085. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":28.205128205128204,"measurements":[{"observationId":"vals-proof-bench:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":22,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.205128205128204,"participants":40,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1177.352. Standardfehler: 4.163. Kosten je Aufgabe in USD: 0.089853. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 1177.352. Standard error: 4.163. Cost per task in USD: 0.089853. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":23.636363636363637,"measurements":[{"observationId":"vals-tax-agent-bench:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":11.503,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.636363636363637,"participants":56,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 376.514. Standardfehler: 1.957. Kosten je Aufgabe in USD: 0.040042. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 376.514. Standard error: 1.957. Cost per task in USD: 0.040042. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":57.89473684210526,"measurements":[{"observationId":"vals-tax-eval-v2:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.794,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.89473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 62.829. Standardfehler: 0.863. Kosten je Aufgabe in USD: 0.002547. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 62.829. Standard error: 0.863. Cost per task in USD: 0.002547. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":34.72222222222222,"measurements":[{"observationId":"vals-vibe-code-bench:overall:xiaomi/mimo-v2.5-pro","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":34.113,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.72222222222222,"participants":73,"methodology":{"de":"Kennung des Quellmodells: xiaomi/mimo-v2.5-pro. Name des Quellmodells: MiMo-V2.5-Pro. Anbieter laut Quelle: Xiaomi. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2403.764. Standardfehler: 4.529. Kosten je Aufgabe in USD: 0.166747. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: xiaomi/mimo-v2.5-pro. Source model name: MiMo-V2.5-Pro. Provider reported by source: Xiaomi. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2403.764. Standard error: 4.529. Cost per task in USD: 0.166747. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":30,"totalFamilies":71,"sourceCount":3,"coverage":0.3201247165532881,"lowerBound":12.248070394367728,"upperBound":80.23559873903892,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":3,"coverage":0.3555555555555556,"lowerBound":11.550057539394313,"upperBound":75.99450198383876,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":19.47093343113882,"upperBound":76.61379057399596,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":5.6410256410256405,"upperBound":85.64102564102564,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":53.215077605321504,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":35.47671840354767,"upperBound":68.81005173688101,"provisional":true,"configurationsDiffer":true},"agentic":{"score":36.273857721226136,"coveredFamilies":6,"totalFamilies":12,"sourceCount":1,"coverage":0.5,"lowerBound":18.136928860613068,"upperBound":68.13692886061307,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":8.88888888888889,"upperBound":88.88888888888889,"provisional":true,"configurationsDiffer":true},"finance":{"score":39.24132469181533,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":28.029517637010947,"upperBound":56.600946208439524,"provisional":true,"configurationsDiffer":true},"legal":{"score":28.24155807762365,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":21.18116855821774,"upperBound":46.181168558217735,"provisional":true,"configurationsDiffer":true},"health":{"score":34.64661984196664,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":23.09774656131109,"upperBound":56.43107989464443,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mimo-v2.5-pro-ultraspeed","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mimo-v2.6-distill-qwen-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mimo-v2.6-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mimo-v2.6-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mimo-v2.6-pro-ultraspeed","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ming-flash-omni-2-0","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"minimax-01","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"minimax-m1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"minimax-m2","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":75.3623188405797,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:f74ea286-cd29-4eb4-af14-1389b19c21e5","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":72.312925170068,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75.3623188405797,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2 (Release minimax-m2, Zeile minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/f74ea286-cd29-4eb4-af14-1389b19c21e5/minimax-m2/minimax-m2. Name des Quellmodells: MiniMax M2. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 72,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2 (release minimax-m2, row minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/f74ea286-cd29-4eb4-af14-1389b19c21e5/minimax-m2/minimax-m2. Source model name: MiniMax M2. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 72.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2 (Release minimax-m2, Zeile minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2 (release minimax-m2, row minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2 (Release minimax-m2, Zeile minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2 (release minimax-m2, row minimax-m2, UUID f74ea286-cd29-4eb4-af14-1389b19c21e5, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":1.0766045548654246,"upperBound":99.64803312629401,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":15.072463768115941,"upperBound":95.07246376811594,"provisional":true,"configurationsDiffer":true}}},{"modelId":"minimax-m2.1","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:272ff333-442f-4169-a804-ac9177bc99d7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":69.8639455782313,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":66.66666666666667,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.1 (Release minimax-m2-1, Zeile minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/272ff333-442f-4169-a804-ac9177bc99d7/minimax-m2-1/minimax-m2-1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 69,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.1 (release minimax-m2-1, row minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/272ff333-442f-4169-a804-ac9177bc99d7/minimax-m2-1/minimax-m2-1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 69.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.1 (Release minimax-m2-1, Zeile minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.1 (release minimax-m2-1, row minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.1 (Release minimax-m2-1, Zeile minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.1 (release minimax-m2-1, row minimax-m2-1, UUID 272ff333-442f-4169-a804-ac9177bc99d7, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":31.57894736842105,"measurements":[{"observationId":"vals-gpqa:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":78.536,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.57894736842105,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 135.412. Standardfehler: 2.128. Kosten je Aufgabe in USD: 0.011422. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 135.412. Standard error: 2.128. Cost per task in USD: 0.011422. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":42.3728813559322,"measurements":[{"observationId":"vals-swebench:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":74.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.3728813559322,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 386.256. Standardfehler: 1.944. Kosten je Aufgabe in USD: 0.439041. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 386.256. Standard error: 1.944. Cost per task in USD: 0.439041. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":35.714285714285715,"measurements":[{"observationId":"vals-terminal-bench-2:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":37.079,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.714285714285715,"participants":43,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 782.07. Standardfehler: 5.149. Kosten je Aufgabe in USD: 0.070371. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 782.07. Standard error: 5.149. Cost per task in USD: 0.070371. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":27.027027027027028,"measurements":[{"observationId":"vals-corp-fin-v2:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":58.896,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.027027027027028,"participants":75,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 21.969. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.022461. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 21.969. Standard error: 0.968. Cost per task in USD: 0.022461. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":44.15584415584416,"measurements":[{"observationId":"vals-lcb:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":81.756,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.15584415584416,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 246.072. Standardfehler: 1.099. Kosten je Aufgabe in USD: 0.018485. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 246.072. Standard error: 1.099. Cost per task in USD: 0.018485. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":14.102564102564102,"measurements":[{"observationId":"vals-legal-bench:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":75.448,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.102564102564102,"participants":79,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 11.491. Standardfehler: 0.539. Kosten je Aufgabe in USD: 0.000937. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 11.491. Standard error: 0.539. Cost per task in USD: 0.000937. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"vals-math500:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":89,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":23,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 59.692. Standardfehler: 1.399. Kosten je Aufgabe in USD: 0.004642. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 59.692. Standard error: 1.399. Cost per task in USD: 0.004642. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":17.91044776119403,"measurements":[{"observationId":"vals-medcode:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":34.083,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.91044776119403,"participants":68,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 19.548. Standardfehler: 1.943. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 19.548. Standard error: 1.943. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"vals-medqa:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":91.158,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":40,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 35.938. Standardfehler: 0.259. Kosten je Aufgabe in USD: 0.002383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 35.938. Standard error: 0.259. Cost per task in USD: 0.002383. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":44.11764705882353,"measurements":[{"observationId":"vals-medscribe:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":80.777,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.11764705882353,"participants":69,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 53.161. Standardfehler: 1.831. Kosten je Aufgabe in USD: 0.005087. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 53.161. Standard error: 1.831. Cost per task in USD: 0.005087. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":27.586206896551722,"measurements":[{"observationId":"vals-mgsm:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":87.854,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.586206896551722,"participants":30,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 13.862. Standardfehler: 1.827. Kosten je Aufgabe in USD: 0.001335. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 13.862. Standard error: 1.827. Cost per task in USD: 0.001335. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":70.12987012987013,"measurements":[{"observationId":"vals-mmlu-pro:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":87.049,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":70.12987012987013,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 40.957. Standardfehler: 0.333. Kosten je Aufgabe in USD: 0.003225. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 40.957. Standard error: 0.333. Cost per task in USD: 0.003225. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":10.526315789473685,"measurements":[{"observationId":"vals-tax-eval-v2:overall:minimax/MiniMax-M2.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":66.353,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.1. Name des Quellmodells: MiniMax M2.1. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 113.258. Standardfehler: 0.925. Kosten je Aufgabe in USD: 0.009137. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.1. Source model name: MiniMax M2.1. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131000. Latency in seconds: 113.258. Standard error: 0.925. Cost per task in USD: 0.009137. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":14,"totalFamilies":71,"sourceCount":2,"coverage":0.1893990929705216,"lowerBound":7.376983948102067,"upperBound":88.43707465104991,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":1,"coverage":0.11111111111111113,"lowerBound":4.752977719079416,"upperBound":93.6418666079683,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":2.255639097744361,"upperBound":95.11278195488723,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":14.608150470219435,"upperBound":74.60815047021944,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":23.376623376623375,"upperBound":90.04329004329004,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":13.333333333333336,"upperBound":93.33333333333334,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":5.364763259500102,"upperBound":76.79333468807154,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":3.5256410256410255,"upperBound":78.52564102564102,"provisional":true,"configurationsDiffer":true},"health":{"score":36.0606469912879,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":36.0606469912879,"upperBound":36.0606469912879,"provisional":true,"configurationsDiffer":true}}},{"modelId":"minimax-m2.5","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":73.91304347826087,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:12adec16-19fe-4d92-aeff-5ef3eb7e780a","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":71.6326530612245,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.91304347826087,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.5 (Release minimax-m2-5, Zeile minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/12adec16-19fe-4d92-aeff-5ef3eb7e780a/minimax-m2-5/minimax-m2-5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 71,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.5 (release minimax-m2-5, row minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/12adec16-19fe-4d92-aeff-5ef3eb7e780a/minimax-m2-5/minimax-m2-5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 71.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.5 (Release minimax-m2-5, Zeile minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.5 (release minimax-m2-5, row minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.5 (Release minimax-m2-5, Zeile minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.5 (release minimax-m2-5, row minimax-m2-5, UUID 12adec16-19fe-4d92-aeff-5ef3eb7e780a, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":40.78947368421053,"measurements":[{"observationId":"vals-gpqa:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":82.07,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.78947368421053,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 171.962. Standardfehler: 2.258. Kosten je Aufgabe in USD: 0.017869. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 171.962. Standard error: 2.258. Cost per task in USD: 0.017869. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":40.67796610169491,"measurements":[{"observationId":"vals-swebench:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":74.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.67796610169491,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 402.941. Standardfehler: 1.959. Kosten je Aufgabe in USD: 0.456928. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 402.941. Standard error: 1.959. Cost per task in USD: 0.456928. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":52.38095238095238,"measurements":[{"observationId":"vals-terminal-bench-2:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":41.573,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.38095238095238,"participants":43,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 754.513. Standardfehler: 5.254. Kosten je Aufgabe in USD: 0.085163. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 754.513. Standard error: 5.254. Cost per task in USD: 0.085163. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":31.08108108108108,"measurements":[{"observationId":"vals-corp-fin-v2:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":59.596,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.08108108108108,"participants":75,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 16.599. Standardfehler: 0.966. Kosten je Aufgabe in USD: 0.024284. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 16.599. Standard error: 0.966. Cost per task in USD: 0.024284. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":29.87012987012987,"measurements":[{"observationId":"vals-lcb:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":79.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.87012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 312.011. Standardfehler: 1.119. Kosten je Aufgabe in USD: 0.033052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 312.011. Standard error: 1.119. Cost per task in USD: 0.033052. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":32.05128205128205,"measurements":[{"observationId":"vals-legal-bench:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.963,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.05128205128205,"participants":79,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 15.819. Standardfehler: 0.473. Kosten je Aufgabe in USD: 0.001287. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 15.819. Standard error: 0.473. Cost per task in USD: 0.001287. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":58.97435897435897,"measurements":[{"observationId":"vals-medqa:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":92.525,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.97435897435897,"participants":40,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 33.54. Standardfehler: 0.24. Kosten je Aufgabe in USD: 0.002928. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 33.54. Standard error: 0.24. Cost per task in USD: 0.002928. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":23.376623376623378,"measurements":[{"observationId":"vals-mmlu-pro:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":80.093,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.376623376623378,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 48.271. Standardfehler: 0.39. Kosten je Aufgabe in USD: 0.005362. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 48.271. Standard error: 0.39. Cost per task in USD: 0.005362. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":18.42105263157895,"measurements":[{"observationId":"vals-tax-eval-v2:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":68.152,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.42105263157895,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 73.243. Standardfehler: 0.906. Kosten je Aufgabe in USD: 0.008629. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 73.243. Standard error: 0.906. Cost per task in USD: 0.008629. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":18.055555555555557,"measurements":[{"observationId":"vals-vibe-code-bench:overall:minimax/MiniMax-M2.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":14.853,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.055555555555557,"participants":73,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.5. Name des Quellmodells: MiniMax M2.5. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 3065.622. Standardfehler: 2.953. Kosten je Aufgabe in USD: 2.197729. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.5. Source model name: MiniMax M2.5. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 3065.622. Standard error: 2.953. Cost per task in USD: 2.197729. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":11,"totalFamilies":71,"sourceCount":2,"coverage":0.11797052154195013,"lowerBound":4.668858025811113,"upperBound":92.8718058716161,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":4,"totalFamilies":15,"sourceCount":1,"coverage":0.1777777777777778,"lowerBound":5.132988480446109,"upperBound":87.35521070266833,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":2.9135338345864663,"upperBound":95.77067669172932,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":7.792207792207793,"upperBound":74.45887445887446,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":14.782608695652174,"upperBound":94.78260869565217,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":7.07173338752286,"upperBound":78.50030481609429,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":8.012820512820513,"upperBound":83.01282051282051,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":19.65811965811966,"upperBound":86.32478632478633,"provisional":true,"configurationsDiffer":true}}},{"modelId":"minimax-m2.7","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":10.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":11.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 11,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 11.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":718.55,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":53.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":78.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 78,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 78.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":11.538461538461538,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":10.6194690265487,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":11.538461538461538,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 10,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 10.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":4.435290250372898,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 4,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 4.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":31.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 6,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 6.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":1010.1,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":86.95652173913044,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":75.7142857142857,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":86.95652173913044,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 75,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 75.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":11.428571428571429,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":26.459510357815404,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":11.428571428571429,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 26,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 26.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":0.7666666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":60.526315789473685,"measurements":[{"observationId":"vals-gpqa:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":86.616,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":60.526315789473685,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 171.727. Standardfehler: 1.953. Kosten je Aufgabe in USD: 0.014051. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 171.727. Standard error: 1.953. Cost per task in USD: 0.014051. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":29.610750695088,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 29,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 29.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":37.28813559322034,"measurements":[{"observationId":"vals-swebench:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":73.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.28813559322034,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 885.69. Standardfehler: 1.968. Kosten je Aufgabe in USD: 0.466052. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 885.69. Standard error: 1.968. Cost per task in USD: 0.466052. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":26.681184668989555,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:4bbceacb-cf47-464b-b60f-e1d1fe016d67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2462:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M2.7. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4bbceacb-cf47-464b-b60f-e1d1fe016d67/minimax-m2-7/minimax-m2-7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Model snapshot: MiniMax-M2.7. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M2.7 (Release minimax-m2-7, Zeile minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M2.7 (release minimax-m2-7, row minimax-m2-7, UUID 4bbceacb-cf47-464b-b60f-e1d1fe016d67, reasoning=true, source reasoning-token configuration budget=2462, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":47.191,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.28571428571429,"participants":43,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 839.048. Standardfehler: 5.322. Kosten je Aufgabe in USD: 0.073995. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 839.048. Standard error: 5.322. Cost per task in USD: 0.073995. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":48.689,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.071428571428573,"participants":57,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 812.638. Standardfehler: 3.066. Kosten je Aufgabe in USD: 0.066012. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 812.638. Standard error: 3.066. Cost per task in USD: 0.066012. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":17.741935483870968,"measurements":[{"observationId":"vals-code-migration:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":8.685,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.741935483870968,"participants":63,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1259.318. Standardfehler: 1.786. Kosten je Aufgabe in USD: 0.587624. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1259.318. Standard error: 1.786. Cost per task in USD: 0.587624. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":49.32432432432432,"measurements":[{"observationId":"vals-corp-fin-v2:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.189,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.32432432432432,"participants":75,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 10.973. Standardfehler: 0.96. Kosten je Aufgabe in USD: 0.022663. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 10.973. Standard error: 0.96. Cost per task in USD: 0.022663. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":13.559322033898304,"measurements":[{"observationId":"vals-emb:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":28.451,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.559322033898304,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1207.259. Standardfehler: 2.431. Kosten je Aufgabe in USD: 1.658686. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1207.259. Standard error: 2.431. Cost per task in USD: 1.658686. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":3.3333333333333335,"measurements":[{"observationId":"vals-finance-agent-v2:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":27.887,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.3333333333333335,"participants":61,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 803.528. Standardfehler: 0.508. Kosten je Aufgabe in USD: 0.18348. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 803.528. Standard error: 0.508. Cost per task in USD: 0.18348. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":31.16883116883117,"measurements":[{"observationId":"vals-lcb:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":79.926,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":31.16883116883117,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 318.934. Standardfehler: 1.096. Kosten je Aufgabe in USD: 0.02142. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 318.934. Standard error: 1.096. Cost per task in USD: 0.02142. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 373.931. Standardfehler: 0. Kosten je Aufgabe in USD: 0.489594. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 373.931. Standard error: 0. Cost per task in USD: 0.489594. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":62.82051282051282,"measurements":[{"observationId":"vals-legal-bench:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.98,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.82051282051282,"participants":79,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 8.916. Standardfehler: 0.411. Kosten je Aufgabe in USD: 0.000626. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 8.916. Standard error: 0.411. Cost per task in USD: 0.000626. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":11.666666666666666,"measurements":[{"observationId":"vals-legal-research:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":10.577,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.666666666666666,"participants":61,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 420.647. Standardfehler: 2.138. Kosten je Aufgabe in USD: 0.219167. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 420.647. Standard error: 2.138. Cost per task in USD: 0.219167. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":19.402985074626866,"measurements":[{"observationId":"vals-medcode:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":34.44,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.402985074626866,"participants":68,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 30.252. Standardfehler: 1.985. Kosten je Aufgabe in USD: 0.007424. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 30.252. Standard error: 1.985. Cost per task in USD: 0.007424. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":39.705882352941174,"measurements":[{"observationId":"vals-medscribe:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":79.867,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.705882352941174,"participants":69,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 27.25. Standardfehler: 1.86. Kosten je Aufgabe in USD: 0.005124. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 27.25. Standard error: 1.86. Cost per task in USD: 0.005124. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":24.675324675324674,"measurements":[{"observationId":"vals-mmlu-pro:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":80.427,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.675324675324674,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 50.005. Standardfehler: 0.39. Kosten je Aufgabe in USD: 0.003947. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 50.005. Standard error: 0.39. Cost per task in USD: 0.003947. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1460.176. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1460.176. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":10.909090909090908,"measurements":[{"observationId":"vals-tax-agent-bench:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":6.872,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.909090909090908,"participants":56,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 302.854. Standardfehler: 1.677. Kosten je Aufgabe in USD: 0.120058. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 302.854. Standard error: 1.677. Cost per task in USD: 0.120058. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":11.842105263157896,"measurements":[{"observationId":"vals-tax-eval-v2:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":66.558,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.842105263157896,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 28.694. Standardfehler: 0.92. Kosten je Aufgabe in USD: 0.003474. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 28.694. Standard error: 0.92. Cost per task in USD: 0.003474. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":13.88888888888889,"measurements":[{"observationId":"vals-vibe-code-bench:overall:minimax/MiniMax-M2.7","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":11.926,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.88888888888889,"participants":73,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M2.7. Name des Quellmodells: MiniMax M2.7. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608. Latenz in Sekunden: 1377.326. Standardfehler: 2.857. Kosten je Aufgabe in USD: 2.819109. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M2.7. Source model name: MiniMax M2.7. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608. Latency in seconds: 1377.326. Standard error: 2.857. Cost per task in USD: 2.819109. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 196608","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 196608"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":29,"totalFamilies":71,"sourceCount":2,"coverage":0.3058390022675737,"lowerBound":9.79217893923299,"upperBound":79.20827871247562,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":2,"coverage":0.3555555555555556,"lowerBound":8.54452265427066,"upperBound":72.98896709871511,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":19.305886667889233,"upperBound":76.44874381074638,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":39.16693063034526,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":26.111287086896844,"upperBound":59.44462042023018,"provisional":true,"configurationsDiffer":true},"agentic":{"score":24.078159752100703,"coveredFamilies":6,"totalFamilies":12,"sourceCount":1,"coverage":0.5,"lowerBound":12.039079876050351,"upperBound":62.03907987605035,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":17.391304347826086,"upperBound":97.3913043478261,"provisional":true,"configurationsDiffer":true},"finance":{"score":17.793635172760954,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":12.709739409114968,"upperBound":41.28116798054355,"provisional":true,"configurationsDiffer":true},"legal":{"score":28.380972397365838,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":21.28572929802438,"upperBound":46.28572929802438,"provisional":true,"configurationsDiffer":true},"health":{"score":29.554433713784018,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":19.702955809189344,"upperBound":53.036289142522676,"provisional":true,"configurationsDiffer":true}}},{"modelId":"minimax-m3","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":7.894736842105263,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":10,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.894736842105263,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 10,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 10.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1090.37,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":88.41463414634147,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":83,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":88.41463414634147,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 83,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 83.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":52.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":21.25125760074941,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":52.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 21,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 21.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":6.666666666666667,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":23.0037666666667,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.666666666666667,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Angezeigter Wert: 23,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Score display: 23.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":9.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 9,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 9.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":59.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1242.47,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":59.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":100,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":82.85714285714289,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":100,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 82,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 82.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":58.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.35,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":58.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":82.23684210526316,"measurements":[{"observationId":"vals-gpqa:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":92.676,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.23684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 278.908. Standardfehler: 1.443. Kosten je Aufgabe in USD: 0.0255. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 278.908. Standard error: 1.443. Cost per task in USD: 0.0255. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":60.97560975609756,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":38.9712696941613,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 39,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 39.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":50.84536474164134,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":78.5549132947977,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.392857142857146,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 78,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 78.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":81.156,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.297872340425535,"participants":48,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 68.354. Standardfehler: 0.94. Kosten je Aufgabe in USD: 0.009431. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 68.354. Standard error: 0.94. Cost per task in USD: 0.009431. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":45.76271186440678,"measurements":[{"observationId":"vals-swebench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":75,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.76271186440678,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 726.041. Standardfehler: 1.938. Kosten je Aufgabe in USD: 0.416355. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 726.041. Standard error: 1.938. Cost per task in USD: 0.416355. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.9999999999999999,"score":44.78113784516224,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":2.02020202020202,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 2,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 2.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":46.067,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.904761904761905,"participants":43,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 1019.457. Standardfehler: 5.314. Kosten je Aufgabe in USD: 0.216209. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 1019.457. Standard error: 5.314. Cost per task in USD: 0.216209. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":53.558,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.678571428571427,"participants":57,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 1023.416. Standardfehler: 0.749. Kosten je Aufgabe in USD: 0.205973. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 1023.416. Standard error: 0.749. Cost per task in USD: 0.205973. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":1.01,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.81081081081081,"participants":38,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 4582.924. Standardfehler: 1.01. Kosten je Aufgabe in USD: 7.229497. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 4582.924. Standard error: 1.01. Cost per task in USD: 7.229497. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":0.5,"score":15.217391304347826,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:277f939a-985b-4b37-859d-b3eabc7c0b26","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.476190476190476,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.217391304347826,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Modellstand: MiniMax-M3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/277f939a-985b-4b37-859d-b3eabc7c0b26/minimax-m3/minimax-m3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Model snapshot: MiniMax-M3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: MiniMax-M3 (Release minimax-m3, Zeile minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: MiniMax-M3 (release minimax-m3, row minimax-m3, UUID 277f939a-985b-4b37-859d-b3eabc7c0b26, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":35.483870967741936,"measurements":[{"observationId":"vals-code-migration:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":19.925,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.483870967741936,"participants":63,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 4428.262. Standardfehler: 3.944. Kosten je Aufgabe in USD: 7.072659. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 4428.262. Standard error: 3.944. Cost per task in USD: 7.072659. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":89.1891891891892,"measurements":[{"observationId":"vals-corp-fin-v2:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":68.104,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.1891891891892,"participants":75,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 24.459. Standardfehler: 0.918. Kosten je Aufgabe in USD: 0.049736. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 24.459. Standard error: 0.918. Cost per task in USD: 0.049736. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":39.39393939393939,"measurements":[{"observationId":"vals-cyberbench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":65,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":39.39393939393939,"participants":34,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 2456.402. Standardfehler: 6.104. Kosten je Aufgabe in USD: 3.408292. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 2456.402. Standard error: 6.104. Cost per task in USD: 3.408292. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":27.11864406779661,"measurements":[{"observationId":"vals-emb:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":47.759,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.11864406779661,"participants":60,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 1890.343. Standardfehler: 2.946. Kosten je Aufgabe in USD: 2.091762. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 1890.343. Standard error: 2.946. Cost per task in USD: 2.091762. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":40,"measurements":[{"observationId":"vals-finance-agent-v2:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":48.269,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40,"participants":61,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 496.894. Standardfehler: 0.437. Kosten je Aufgabe in USD: 0.32345. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 496.894. Standard error: 0.437. Cost per task in USD: 0.32345. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":49.35064935064935,"measurements":[{"observationId":"vals-lcb:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.154,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.35064935064935,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 366.506. Standardfehler: 1.055. Kosten je Aufgabe in USD: 0.03743. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 366.506. Standard error: 1.055. Cost per task in USD: 0.03743. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":59.01639344262295,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":4.167,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.01639344262295,"participants":62,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 1351.134. Standardfehler: 1.646. Kosten je Aufgabe in USD: 1.460248. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 1351.134. Standard error: 1.646. Cost per task in USD: 1.460248. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":79.48717948717949,"measurements":[{"observationId":"vals-legal-bench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":85.415,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.48717948717949,"participants":79,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 8.205. Standardfehler: 0.436. Kosten je Aufgabe in USD: 0.00091. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 8.205. Standard error: 0.436. Cost per task in USD: 0.00091. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":43.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":29.808,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 813.838. Standardfehler: 3.179. Kosten je Aufgabe in USD: 0.342955. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 813.838. Standard error: 3.179. Cost per task in USD: 0.342955. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":59.701492537313435,"measurements":[{"observationId":"vals-medcode:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":46.289,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":59.701492537313435,"participants":68,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 63.12. Standardfehler: 2.104. Kosten je Aufgabe in USD: 0.012125. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 63.12. Standard error: 2.104. Cost per task in USD: 0.012125. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":82.3529411764706,"measurements":[{"observationId":"vals-medscribe:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":87.252,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.3529411764706,"participants":69,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 123.967. Standardfehler: 1.957. Kosten je Aufgabe in USD: 0.013748. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 123.967. Standard error: 1.957. Cost per task in USD: 0.013748. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":41.55844155844156,"measurements":[{"observationId":"vals-mmlu-pro:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":84.22,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.55844155844156,"participants":78,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 41.39. Standardfehler: 0.364. Kosten je Aufgabe in USD: 0.004811. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 41.39. Standard error: 0.364. Cost per task in USD: 0.004811. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"vals-mortgage-tax:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.362,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":53,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 26.756. Standardfehler: 0.91. Kosten je Aufgabe in USD: 0.008326. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 26.756. Standard error: 0.91. Cost per task in USD: 0.008326. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":23.076923076923077,"measurements":[{"observationId":"vals-proof-bench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":18,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.076923076923077,"participants":40,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 718.66. Standardfehler: 3.861. Kosten je Aufgabe in USD: 0.419224. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 718.66. Standard error: 3.861. Cost per task in USD: 0.419224. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":79.3103448275862,"measurements":[{"observationId":"vals-sage:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":50.574,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.3103448275862,"participants":59,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 118.538. Standardfehler: 3.436. Kosten je Aufgabe in USD: 0.025844. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 118.538. Standard error: 3.436. Cost per task in USD: 0.025844. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":48.275862068965516,"measurements":[{"observationId":"vals-skillsbench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":51.497,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":48.275862068965516,"participants":30,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 927.977. Standardfehler: 4.498. Kosten je Aufgabe in USD: 0.504553. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 927.977. Standard error: 4.498. Cost per task in USD: 0.504553. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":55.26315789473684,"measurements":[{"observationId":"vals-snap:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":64.141,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.26315789473684,"participants":39,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 691.436. Standardfehler: 1.247. Kosten je Aufgabe in USD: 0.2543. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 691.436. Standard error: 1.247. Cost per task in USD: 0.2543. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":40,"measurements":[{"observationId":"vals-tax-agent-bench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":20.588,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40,"participants":56,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 314.575. Standardfehler: 2.874. Kosten je Aufgabe in USD: 0.164063. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 314.575. Standard error: 2.874. Cost per task in USD: 0.164063. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":50,"measurements":[{"observationId":"vals-tax-eval-v2:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":72.731,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":77,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 94.036. Standardfehler: 0.863. Kosten je Aufgabe in USD: 0.00728. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 94.036. Standard error: 0.863. Cost per task in USD: 0.00728. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":9.523809523809524,"measurements":[{"observationId":"vals-vcb-1-100:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":9.222,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.523809523809524,"participants":22,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 1570.129. Standardfehler: 2.857. Kosten je Aufgabe in USD: 3.028103. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 1570.129. Standard error: 2.857. Cost per task in USD: 3.028103. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":43.05555555555556,"measurements":[{"observationId":"vals-vibe-code-bench:overall:minimax/MiniMax-M3","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":47.566,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.05555555555556,"participants":73,"methodology":{"de":"Kennung des Quellmodells: minimax/MiniMax-M3. Name des Quellmodells: MiniMax M3. Anbieter laut Quelle: MiniMax. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000. Latenz in Sekunden: 5073.23. Standardfehler: 5.442. Kosten je Aufgabe in USD: 6.453899. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: minimax/MiniMax-M3. Source model name: MiniMax M3. Provider reported by source: MiniMax. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000. Latency in seconds: 5073.23. Standard error: 5.442. Cost per task in USD: 6.453899. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 512000","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 512000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":36,"totalFamilies":71,"sourceCount":2,"coverage":0.49302721088435386,"lowerBound":27.075134652906858,"upperBound":77.77241356447148,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":8,"totalFamilies":15,"sourceCount":2,"coverage":0.4666666666666669,"lowerBound":14.623449432930842,"upperBound":67.95678276626415,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":28.957683843755728,"upperBound":86.10054098661287,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":4.615384615384616,"upperBound":84.61538461538461,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":50.047513462147606,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":33.36500897476507,"upperBound":66.6983423080984,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":19.11177755153232,"upperBound":77.44511088486564,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":20,"upperBound":100,"provisional":true,"configurationsDiffer":true},"finance":{"score":53.87181836334379,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":46.17584431143753,"upperBound":60.46155859715182,"provisional":true,"configurationsDiffer":true},"legal":{"score":60.612302087711925,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":45.45922656578394,"upperBound":70.45922656578395,"provisional":true,"configurationsDiffer":true},"health":{"score":71.02721685689201,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":47.35147790459468,"upperBound":80.68481123792802,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":39.39393939393939,"upperBound":39.39393939393939,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":12.711341185410335,"upperBound":87.71134118541033,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":27.63157894736842,"upperBound":77.63157894736842,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":39.6551724137931,"upperBound":89.65517241379311,"provisional":true,"configurationsDiffer":true}}},{"modelId":"minimax-m3.1-flash-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"minimax-minimax-m2.5-highspeed","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"minimax-minimax-m2.7-highspeed","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ministral-3-14b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":131.09,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":4.878048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":26.333333333333297,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.878048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 26,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 26.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.5512132673736124,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":247.72,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":10.869565217391305,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":32.0408163265306,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.869565217391305,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 32,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 32.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":1.2195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-66.36666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.2195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.6339202965709,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":8.928571428571429,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":49.8265895953757,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.928571428571429,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 49,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 49.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:713fae11-c75c-4f10-ae2c-8e4074cd58af","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Name des Quellmodells: Ministral 3 14B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 14B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/713fae11-c75c-4f10-ae2c-8e4074cd58af/ministral-3-14b/ministral-3-14b. Source model name: Ministral 3 14B. Provider reported by source: Mistral. Model snapshot: Ministral 3 14B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 14B (Release ministral-3-14b, Zeile ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 14B (release ministral-3-14b, row ministral-3-14b, UUID 713fae11-c75c-4f10-ae2c-8e4074cd58af, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.09277210884353743,"lowerBound":0.7012341654583361,"upperBound":91.4240232811046,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":2.787456445993031,"upperBound":67.07317073170732,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0.4065040650406504,"upperBound":67.07317073170732,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":2.7439024390243905,"upperBound":77.7439024390244,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.1739130434782608,"upperBound":82.17391304347827,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":1.1160714285714286,"upperBound":88.61607142857143,"provisional":true,"configurationsDiffer":true}}},{"modelId":"ministral-3-3b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":17,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 17,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 17.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.6658657772309807,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":1.829268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.829268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":4.878048780487805,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":29.62,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.878048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":2.898550724637681,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":26.802721088435398,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.898550724637681,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 26,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 26.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":2.4390243902439024,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-64,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.4390243902439024,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":5.37534754402224,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 5,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 5.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":38.092485549132896,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 38,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 38.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Name des Quellmodells: Ministral 3 3B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 3B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720/ministral-3-3b/ministral-3-3b. Source model name: Ministral 3 3B. Provider reported by source: Mistral. Model snapshot: Ministral 3 3B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 3B (Release ministral-3-3b, Zeile ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 3B (release ministral-3-3b, row ministral-3-3b, UUID 66f4ce73-9a9b-4b49-9c6e-bedb9bfdc720, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.09277210884353743,"lowerBound":0.34327861579414376,"upperBound":91.06606773144041,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":1.0452961672473868,"upperBound":65.33101045296168,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0.8130081300813008,"upperBound":67.47967479674797,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":1.7784552845528454,"upperBound":76.77845528455285,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.5797101449275363,"upperBound":80.57971014492753,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":0,"upperBound":87.5,"provisional":true,"configurationsDiffer":true}}},{"modelId":"ministral-3-8b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":77.44,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":3.658536585365854,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":25.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.658536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 25,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 25.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.516749466666553,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":214.05,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":5.797101449275362,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":29.115646258503403,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":5.797101449275362,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 29,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 29.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-68.95,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":3.658536585365854,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.2632066728452305,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.658536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":5.357142857142857,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":45.9537572254335,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":5.357142857142857,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 46,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 46.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:9741f3c2-cbb1-4a3f-99ee-7bd7384d9038","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Name des Quellmodells: Ministral 3 8B. Anbieter laut Quelle: Mistral. Modellstand: Ministral 3 8B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/9741f3c2-cbb1-4a3f-99ee-7bd7384d9038/ministral-3-8b/ministral-3-8b. Source model name: Ministral 3 8B. Provider reported by source: Mistral. Model snapshot: Ministral 3 8B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ministral 3 8B (Release ministral-3-8b, Zeile ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Ministral 3 8B (release ministral-3-8b, row ministral-3-8b, UUID 9741f3c2-cbb1-4a3f-99ee-7bd7384d9038, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.09277210884353743,"lowerBound":0.4361476165227491,"upperBound":91.15893673216901,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":1.6550522648083625,"upperBound":65.94076655052265,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0,"upperBound":66.66666666666667,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":2.032520325203252,"upperBound":77.03252032520325,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.1594202898550725,"upperBound":81.15942028985508,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":0.6696428571428571,"upperBound":88.16964285714286,"provisional":true,"configurationsDiffer":true}}},{"modelId":"ministral-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-codestral-22b-v0.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-codestral-latest","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-devstral-2512","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-devstral-small-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-devstral-small-2507","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-large-3","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":228.3,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":36,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 36,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 36.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.4974701708193463,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":420.4,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":18.840579710144926,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":36.190476190476204,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.840579710144926,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 36,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 36.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-39.56666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":17.105263157894736,"measurements":[{"observationId":"vals-gpqa:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":68.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.105263157894736,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 21.216. Standardfehler: 2.372. Kosten je Aufgabe in USD: 0.00215. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 21.216. Standard error: 2.372. Cost per task in USD: 0.00215. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":2.4390243902439024,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.17052826691381,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.4390243902439024,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":0,"measurements":[{"observationId":"lmarena:webdev:overall:mistral-large-3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1230.2046332670132,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":0,"participants":37,"methodology":{"de":"Arena-Bewertung aus 839 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-large-3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 839 published pairwise votes.. Source model ID: lmarena/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Model snapshot: mistral-large-3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 839 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 839 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 839 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 839 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"lmarena:overall:mistral-large-3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1414.1358256255128,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":0,"participants":37,"methodology":{"de":"Arena-Bewertung aus 83.498 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-large-3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 83,498 published pairwise votes.. Source model ID: lmarena/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Model snapshot: mistral-large-3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 83.498 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 83,498 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 83.498 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 83,498 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":0,"measurements":[{"observationId":"lmarena:vision_style_control:overall:mistral-large-3","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1199.757077684447,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":0,"participants":31,"methodology":{"de":"Arena-Bewertung aus 8.643 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-large-3. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 8,643 published pairwise votes.. Source model ID: lmarena/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Model snapshot: mistral-large-3. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 8.643 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 8,643 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 8.643 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 8,643 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":11.569148936170212,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":55.6647398843931,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.5,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 55,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 55.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":66.185,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.638297872340425,"participants":48,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 13.55. Standardfehler: 1.137. Kosten je Aufgabe in USD: 0.001477. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 13.55. Standard error: 1.137. Cost per task in USD: 0.001477. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":3.389830508474576,"measurements":[{"observationId":"vals-swebench:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":41.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.389830508474576,"participants":60,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 186.955. Standardfehler: 2.205. Kosten je Aufgabe in USD: 0.321666. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 186.955. Standard error: 2.205. Cost per task in USD: 0.321666. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":13.857433217189316,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:4928e950-7f37-4475-b0dc-c5bad781a321","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/4928e950-7f37-4475-b0dc-c5bad781a321/mistral-large-3/mistral-large-3. Source model name: Mistral Large 3. Provider reported by source: Mistral. Model snapshot: Mistral Large 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 3 (Release mistral-large-3, Zeile mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Large 3 (release mistral-large-3, row mistral-large-3, UUID 4928e950-7f37-4475-b0dc-c5bad781a321, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":8.989,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.380952380952381,"participants":43,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1180.302. Standardfehler: 3.049. Kosten je Aufgabe in USD: 0.030482. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 1180.302. Standard error: 3.049. Cost per task in USD: 0.030482. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":45.945945945945944,"measurements":[{"observationId":"vals-corp-fin-v2:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":61.033,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":45.945945945945944,"participants":75,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 48.842. Standardfehler: 0.96. Kosten je Aufgabe in USD: 0.024166. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 48.842. Standard error: 0.96. Cost per task in USD: 0.024166. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":12.987012987012987,"measurements":[{"observationId":"vals-lcb:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":55.337,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.987012987012987,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 28.888. Standardfehler: 1.152. Kosten je Aufgabe in USD: 0.002215. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 28.888. Standard error: 1.152. Cost per task in USD: 0.002215. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":25.641025641025642,"measurements":[{"observationId":"vals-legal-bench:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.138,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.641025641025642,"participants":79,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1.731. Standardfehler: 0.46. Kosten je Aufgabe in USD: 0.000439. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 1.731. Standard error: 0.46. Cost per task in USD: 0.000439. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":25.641025641025642,"measurements":[{"observationId":"vals-medqa:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":82.233,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.641025641025642,"participants":40,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 17.31. Standardfehler: 0.349. Kosten je Aufgabe in USD: 0.000771. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 17.31. Standard error: 0.349. Cost per task in USD: 0.000771. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":6.896551724137931,"measurements":[{"observationId":"vals-mgsm:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":85.418,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.896551724137931,"participants":30,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 7.622. Standardfehler: 1.787. Kosten je Aufgabe in USD: 0.000472. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 7.622. Standard error: 1.787. Cost per task in USD: 0.000472. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":22.07792207792208,"measurements":[{"observationId":"vals-mmlu-pro:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":79.823,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.07792207792208,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 13.554. Standardfehler: 0.472. Kosten je Aufgabe in USD: 0.001481. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 13.554. Standard error: 0.472. Cost per task in USD: 0.001481. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":5.769230769230769,"measurements":[{"observationId":"vals-mortgage-tax:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":52.106,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.769230769230769,"participants":53,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 9.292. Standardfehler: 0.995. Kosten je Aufgabe in USD: 0.003226. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 9.292. Standard error: 0.995. Cost per task in USD: 0.003226. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":1.7241379310344827,"measurements":[{"observationId":"vals-sage:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":24.595,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.7241379310344827,"participants":59,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 55.068. Standardfehler: 2.81. Kosten je Aufgabe in USD: 0.00806. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 55.068. Standard error: 2.81. Cost per task in USD: 0.00806. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":52.63157894736842,"measurements":[{"observationId":"vals-tax-eval-v2:overall:mistralai/mistral-large-2512","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.058,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.63157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2512. Name des Quellmodells: Mistral Large 3. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.065. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 29.531. Standardfehler: 0.867. Kosten je Aufgabe in USD: 0.001601. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2512. Source model name: Mistral Large 3. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.065. Maximum output tokens: 256000. Latency in seconds: 29.531. Standard error: 0.867. Cost per task in USD: 0.001601. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.065. Maximale Ausgabetoken: 256000","en":"Temperature: 0.065. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":24,"totalFamilies":71,"sourceCount":3,"coverage":0.29773242630385494,"lowerBound":4.6033390223081465,"upperBound":74.83009639192265,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":4,"totalFamilies":15,"sourceCount":3,"coverage":0.2111111111111112,"lowerBound":1.5946811368473217,"upperBound":80.4835700257362,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":4.53190904089492,"upperBound":61.674766183752055,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.379310344827586,"upperBound":81.37931034482759,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":22.0145707950586,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":14.676380530039067,"upperBound":48.009713863372404,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":4.878048780487805,"upperBound":79.8780487804878,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":3.7681159420289854,"upperBound":63.768115942028984,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":14.90667938036359,"upperBound":72.04953652322072,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":6.410256410256411,"upperBound":81.41025641025641,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":8.547008547008549,"upperBound":75.21367521367522,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":5.784574468085106,"coveredFamilies":2,"totalFamilies":4,"sourceCount":3,"coverage":0.5,"lowerBound":2.892287234042553,"upperBound":52.892287234042556,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":0.8620689655172413,"upperBound":50.86206896551724,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-large-4","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":76.82926829268293,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1391.97,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":76.82926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":77.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":81.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":77.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 81,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 81.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":84.14634146341463,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":59.90136246808686,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":84.14634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 59,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 59.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":18.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 18,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 18.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1422.17,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":49.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-5.3,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":51.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":35.032437442075995,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 35,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 35.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":51.785714285714285,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":76.4161849710983,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":51.785714285714285,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 76,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 76.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":75.58915622940013,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:34fc646a-0174-470e-9e62-d8df452210b3","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":26.7676767676768,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":82.3170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 26,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/34fc646a-0174-470e-9e62-d8df452210b3/mistral-large-4/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral. Model snapshot: Mistral Large 4 Preview. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 26.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Large 4 Preview (Release mistral-large-4, Zeile mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Large 4 Preview (release mistral-large-4, row mistral-large-4, UUID 34fc646a-0174-470e-9e62-d8df452210b3, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":23.232,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.4054054054054,"participants":38,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 3352.007. Standardfehler: 1.336. Kosten je Aufgabe in USD: 7.897179. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 3352.007. Standard error: 1.336. Cost per task in USD: 7.897179. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-biomysterybench","area":"general","label":{"de":"BioMysteryBench","en":"BioMysteryBench"},"weight":0.014285714285714285,"coverage":1,"score":28.94736842105263,"measurements":[{"observationId":"vals-biomysterybench:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","benchmarkId":"vals-biomysterybench","benchmarkVersion":"1","comparisonCohort":"vals-biomysterybench:1:overall","retrievedAt":"2026-10-15","value":67.037,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.94736842105263,"participants":20,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1532.221. Standardfehler: 2.671. Kosten je Aufgabe in USD: 2.067705. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-biomysterybench-2026-10-15-59bf79d38ec4. Prüfsumme der Quelldaten: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 1532.221. Standard error: 2.671. Cost per task in USD: 2.067705. Source ID: vals-ai. Evidence capture ID: vals-biomysterybench-2026-10-15-59bf79d38ec4. Source payload hash: 59bf79d38ec415a34ffb1c2901a16ecbbe101b7fa864e86104d2571fc7cfc23d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: Terminus 2. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: Terminus 2. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-biomysterybench:1:overall","datasetType":"public","harness":"Terminus 2","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":51.61290322580645,"measurements":[{"observationId":"vals-code-migration:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":30.561,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.61290322580645,"participants":63,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 17441.256. Standardfehler: 4.221. Kosten je Aufgabe in USD: 47.890583. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 17441.256. Standard error: 4.221. Cost per task in USD: 47.890583. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":40.67796610169491,"measurements":[{"observationId":"vals-emb:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":56.213,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.67796610169491,"participants":60,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 4904.981. Standardfehler: 3.22. Kosten je Aufgabe in USD: 4.839324. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 4904.981. Standard error: 3.22. Cost per task in USD: 4.839324. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"vals-finance-agent-v2:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":54.678,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":61,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1561.879. Standardfehler: 0.58. Kosten je Aufgabe in USD: 1.201399. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 1561.879. Standard error: 0.58. Cost per task in USD: 1.201399. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-ioi:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":45.278,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":34,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 5023.763. Standardfehler: 3.783. Kosten je Aufgabe in USD: 5.392691. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 5023.763. Standard error: 3.783. Cost per task in USD: 5.392691. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":94.26229508196721,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":15.833,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":94.26229508196721,"participants":62,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 2364.651. Standardfehler: 2.958. Kosten je Aufgabe in USD: 4.181392. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 2364.651. Standard error: 2.958. Cost per task in USD: 4.181392. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":50,"measurements":[{"observationId":"vals-legal-research:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":31.731,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":61,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 5571.099. Standardfehler: 3.235. Kosten je Aufgabe in USD: 5.32689. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 5571.099. Standard error: 3.235. Cost per task in USD: 5.32689. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":34.32835820895522,"measurements":[{"observationId":"vals-medcode:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.685,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.32835820895522,"participants":68,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 68.682. Standardfehler: 2.171. Kosten je Aufgabe in USD: 0.041909. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 68.682. Standard error: 2.171. Cost per task in USD: 0.041909. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":41.1764705882353,"measurements":[{"observationId":"vals-medscribe:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":80.434,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.1764705882353,"participants":69,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 125.352. Standardfehler: 1.996. Kosten je Aufgabe in USD: 0.047445. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 125.352. Standard error: 1.996. Cost per task in USD: 0.047445. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"vals-mysterymechanism:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":12.162,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":21,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 2136.737. Standardfehler: 2.199. Kosten je Aufgabe in USD: 2.10023. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 2136.737. Standard error: 2.199. Cost per task in USD: 2.10023. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":12.820512820512821,"measurements":[{"observationId":"vals-proof-bench:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":10,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.820512820512821,"participants":40,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1258.806. Standardfehler: 3.015. Kosten je Aufgabe in USD: 0.642365. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 1258.806. Standard error: 3.015. Cost per task in USD: 0.642365. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":67.27272727272727,"measurements":[{"observationId":"vals-tax-agent-bench:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":30.391,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.27272727272727,"participants":56,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 3438.427. Standardfehler: 3.309. Kosten je Aufgabe in USD: 2.72124. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 3438.427. Standard error: 3.309. Cost per task in USD: 2.72124. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":23.80952380952381,"measurements":[{"observationId":"vals-vcb-1-100:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":14.261,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.80952380952381,"participants":22,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 3593.189. Standardfehler: 3.205. Kosten je Aufgabe in USD: 6.749045. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 3593.189. Standard error: 3.205. Cost per task in USD: 6.749045. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":72.22222222222223,"measurements":[{"observationId":"vals-vibe-code-bench:overall:mistralai/mistral-large-4","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":78.403,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.22222222222223,"participants":73,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-4. Name des Quellmodells: Mistral Large 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 10395.145. Standardfehler: 3.55. Kosten je Aufgabe in USD: 39.456533. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-4. Source model name: Mistral Large 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000. Latency in seconds: 10395.145. Standard error: 3.55. Cost per task in USD: 39.456533. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":23,"totalFamilies":71,"sourceCount":2,"coverage":0.25513038548752837,"lowerBound":12.714571866096046,"upperBound":87.2015333173432,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.3111111111111112,"lowerBound":15.020676489991994,"upperBound":83.90956537888087,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":25.217770034843202,"upperBound":89.50348432055749,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.5641025641025643,"upperBound":82.56410256410257,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.463414634146343,"upperBound":83.13008130081302,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":19.10569105691057,"upperBound":94.10569105691057,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":5.7894736842105265,"upperBound":65.78947368421052,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":26.135813339203164,"upperBound":83.2786704820603,"provisional":true,"configurationsDiffer":true},"legal":{"score":72.1311475409836,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":36.0655737704918,"upperBound":86.0655737704918,"provisional":true,"configurationsDiffer":true},"health":{"score":37.752414398595256,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":25.16827626573017,"upperBound":58.50160959906351,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":6.473214285714286,"upperBound":93.97321428571429,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-magistral-small-2506","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-magistral-small-2509","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-medium-3","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":13.157894736842104,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":12.5,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.157894736842104,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 12,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 12.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":522.97,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":69.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 69,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 69.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":37.80487804878049,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":6.299648656993127,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":37.80487804878049,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 6,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 6.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":2.8000000000000003,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 2,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 2.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":760.86,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":63.768115942028984,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":68.7755102040816,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":63.768115942028984,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 68,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 68.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-36.8,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":1.3157894736842106,"measurements":[{"observationId":"vals-gpqa:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":34.848,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3157894736842106,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 256.17. Standardfehler: 2.39. Kosten je Aufgabe in USD: 0.095489. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 256.17. Standard error: 2.39. Cost per task in USD: 0.095489. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":13.7627432808156,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 13,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 13.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":2.7777777777777777,"measurements":[{"observationId":"lmarena:webdev:overall:mistral-medium-3.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1263.3829384837534,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":2.7777777777777777,"participants":37,"methodology":{"de":"Arena-Bewertung aus 2.320 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-medium-3.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 2,320 published pairwise votes.. Source model ID: lmarena/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Model snapshot: mistral-medium-3.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 2.320 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 2,320 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 2.320 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 2,320 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-text-style-control","area":"general","label":{"de":"LMArena Text, Style Control","en":"LMArena Text, Style Control"},"weight":0.014285714285714285,"coverage":1,"score":2.7777777777777777,"measurements":[{"observationId":"lmarena:overall:mistral-medium-3.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/text_style_control","benchmarkId":"lmarena-text-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-text-style-control:2026-10-13:overall","retrievedAt":"2026-10-15","value":1426.6051088690308,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":2.7777777777777777,"participants":37,"methodology":{"de":"Arena-Bewertung aus 11.793 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-medium-3.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Text, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Prüfsumme der Quelldaten: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 11,793 published pairwise votes.. Source model ID: lmarena/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Model snapshot: mistral-medium-3.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Text, style control. Source ID: lmarena. Evidence capture ID: lmarena-4589dee83b594891961ae5190c508a9288d2c2ae-a39cbed99996. Source payload hash: a39cbed99996f4993a4cd66d8322b2edb939f1ffdbd377c9d3769edd01dd78ca. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 11.793 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 11,793 published pairwise votes.. Harness: Arena Text, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 11.793 veröffentlichten Paarvergleichen.. Harness: Arena Text, style control","en":"Arena rating from 11,793 published pairwise votes.. Harness: Arena Text, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-text-style-control:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Text, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"lmarena-vision-style-control","area":"multimodal","label":{"de":"LMArena Vision, Style Control","en":"LMArena Vision, Style Control"},"weight":0.017857142857142856,"coverage":1,"score":3.3333333333333335,"measurements":[{"observationId":"lmarena:vision_style_control:overall:mistral-medium-3.5","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/vision_style_control","benchmarkId":"lmarena-vision-style-control","benchmarkVersion":"style-control-v1","comparisonCohort":"lmarena-vision-style-control:2026-10-09:overall","retrievedAt":"2026-10-15","value":1200.3859487909897,"unit":"rating","metric":"style-controlled Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":3.3333333333333335,"participants":31,"methodology":{"de":"Arena-Bewertung aus 5.375 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Modellstand: mistral-medium-3.5. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-09. Harness: Arena Vision, style control. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Prüfsumme der Quelldaten: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 5,375 published pairwise votes.. Source model ID: lmarena/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Model snapshot: mistral-medium-3.5. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-09. Harness: Arena Vision, style control. Source ID: lmarena. Evidence capture ID: lmarena-vision_style_control-4589dee83b594891961ae5190c508a9288d2c2ae-6cf1fae8efc9. Source payload hash: 6cf1fae8efc9ef6fe066f27d5bbe558cd360803a9c0fa6d89905aa53934efdb4. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 5.375 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 5,375 published pairwise votes.. Harness: Arena Vision, style control"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 5.375 veröffentlichten Paarvergleichen.. Harness: Arena Vision, style control","en":"Arena rating from 5,375 published pairwise votes.. Harness: Arena Vision, style control"},"scoreComparisonSetup":{"publicationCohort":"lmarena-vision-style-control:2026-10-09:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Vision, style control","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":25,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":64.8554913294798,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 64,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 64.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":15.254237288135593,"measurements":[{"observationId":"vals-swebench:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":66.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.254237288135593,"participants":60,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1143.68. Standardfehler: 2.114. Kosten je Aufgabe in USD: 4.004623. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1143.68. Standard error: 2.114. Cost per task in USD: 4.004623. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":17.157375145180023,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:dd059b25-d82a-4ead-82a4-4adceaaec48b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral. Modellstand: Mistral Medium 3.5. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/dd059b25-d82a-4ead-82a4-4adceaaec48b/mistral-medium-3-5/mistral-medium-3-5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral. Model snapshot: Mistral Medium 3.5. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Medium 3.5 (Release mistral-medium-3-5, Zeile mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Mistral Medium 3.5 (release mistral-medium-3-5, row mistral-medium-3-5, UUID dd059b25-d82a-4ead-82a4-4adceaaec48b, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":30.337,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.38095238095238,"participants":43,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1253.133. Standardfehler: 4.901. Kosten je Aufgabe in USD: 4.710448. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1253.133. Standard error: 4.901. Cost per task in USD: 4.710448. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":38.951,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.357142857142857,"participants":57,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 989.169. Standardfehler: 2.701. Kosten je Aufgabe in USD: 1.190984. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 989.169. Standard error: 2.701. Cost per task in USD: 1.190984. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":9.67741935483871,"measurements":[{"observationId":"vals-code-migration:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":5.127,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.67741935483871,"participants":63,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 13055.328. Standardfehler: 1.369. Kosten je Aufgabe in USD: 18.623756. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 13055.328. Standard error: 1.369. Cost per task in USD: 18.623756. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":25.675675675675677,"measurements":[{"observationId":"vals-corp-fin-v2:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":58.78,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.675675675675677,"participants":75,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 43.699. Standardfehler: 0.969. Kosten je Aufgabe in USD: 0.14574. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 43.699. Standard error: 0.969. Cost per task in USD: 0.14574. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":3.389830508474576,"measurements":[{"observationId":"vals-emb:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":11.185,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.389830508474576,"participants":60,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 3082.124. Standardfehler: 1.637. Kosten je Aufgabe in USD: 10.603072. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 3082.124. Standard error: 1.637. Cost per task in USD: 10.603072. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":10,"measurements":[{"observationId":"vals-finance-agent-v2:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":32.103,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10,"participants":61,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 768.405. Standardfehler: 0.967. Kosten je Aufgabe in USD: 1.908513. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 768.405. Standard error: 0.967. Cost per task in USD: 1.908513. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":24.59016393442623,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.59016393442623,"participants":62,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 3694.945. Standardfehler: 0. Kosten je Aufgabe in USD: 6.074265. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 3694.945. Standard error: 0. Cost per task in USD: 6.074265. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":9.166666666666666,"measurements":[{"observationId":"vals-legal-research:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":9.135,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.166666666666666,"participants":61,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 613.248. Standardfehler: 2.002. Kosten je Aufgabe in USD: 1.308079. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 613.248. Standard error: 2.002. Cost per task in USD: 1.308079. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":14.925373134328359,"measurements":[{"observationId":"vals-medcode:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":33.752,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.925373134328359,"participants":68,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 34.879. Standardfehler: 1.148. Kosten je Aufgabe in USD: 0.05337. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 34.879. Standard error: 1.148. Cost per task in USD: 0.05337. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":7.352941176470588,"measurements":[{"observationId":"vals-medscribe:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":67.728,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.352941176470588,"participants":69,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 104.053. Standardfehler: 2.011. Kosten je Aufgabe in USD: 0.157641. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 104.053. Standard error: 2.011. Cost per task in USD: 0.157641. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":11.688311688311689,"measurements":[{"observationId":"vals-mmlu-pro:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":75.335,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.688311688311689,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 66.384. Standardfehler: 0.418. Kosten je Aufgabe in USD: 0.041763. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 66.384. Standard error: 0.418. Cost per task in USD: 0.041763. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-mortgage-tax:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":28.895,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":53,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 68.43. Standardfehler: 0.838. Kosten je Aufgabe in USD: 0.024032. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 68.43. Standard error: 0.838. Cost per task in USD: 0.024032. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":10.256410256410257,"measurements":[{"observationId":"vals-proof-bench:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":9,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.256410256410257,"participants":40,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 479.048. Standardfehler: 2.876. Kosten je Aufgabe in USD: 1.302032. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 479.048. Standard error: 2.876. Cost per task in USD: 1.302032. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":18.96551724137931,"measurements":[{"observationId":"vals-sage:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":37.613,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.96551724137931,"participants":59,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 55.844. Standardfehler: 3.356. Kosten je Aufgabe in USD: 0.072775. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 55.844. Standard error: 3.356. Cost per task in USD: 0.072775. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":5.454545454545454,"measurements":[{"observationId":"vals-tax-agent-bench:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":4.482,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.454545454545454,"participants":56,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 430.673. Standardfehler: 1.47. Kosten je Aufgabe in USD: 1.443655. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 430.673. Standard error: 1.47. Cost per task in USD: 1.443655. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":17.105263157894736,"measurements":[{"observationId":"vals-tax-eval-v2:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":67.988,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.105263157894736,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 102.392. Standardfehler: 0.916. Kosten je Aufgabe in USD: 0.079101. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 102.392. Standard error: 0.916. Cost per task in USD: 0.079101. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":5.555555555555555,"measurements":[{"observationId":"vals-vibe-code-bench:overall:mistralai/mistral-medium-3.5","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":2.888,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.555555555555555,"participants":73,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-medium-3.5. Name des Quellmodells: Mistral Medium 3.5. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 2470.533. Standardfehler: 1.713. Kosten je Aufgabe in USD: 16.735503. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-medium-3.5. Source model name: Mistral Medium 3.5. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 2470.533. Standard error: 1.713. Cost per task in USD: 16.735503. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":31,"totalFamilies":71,"sourceCount":3,"coverage":0.3725907029478459,"lowerBound":6.07453445699718,"upperBound":68.81546416221259,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":3,"coverage":0.288888888888889,"lowerBound":2.66237893021488,"upperBound":73.77349004132599,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":7.759490188886852,"upperBound":64.902347331744,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.0512820512820515,"upperBound":82.05128205128206,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":18.03927779537536,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":12.026185196916906,"upperBound":45.35951853025024,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.124946512623021,"upperBound":75.7916131792897,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":2,"coverage":0.4,"lowerBound":13.309178743961352,"upperBound":73.30917874396135,"provisional":true,"configurationsDiffer":true},"finance":{"score":10.270885799431742,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":8.803616399512922,"upperBound":23.08933068522721,"provisional":true,"configurationsDiffer":true},"legal":{"score":16.87841530054645,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":8.439207650273225,"upperBound":58.439207650273225,"provisional":true,"configurationsDiffer":true},"health":{"score":11.139157155399475,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":7.426104770266316,"upperBound":40.75943810359965,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.375,"lowerBound":3.9583333333333335,"upperBound":66.45833333333333,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":9.482758620689655,"upperBound":59.48275862068965,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-ministral-8b-instruct-2410","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-mistral-large-2411","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":6.578947368421052,"measurements":[{"observationId":"vals-gpqa:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":47.728,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.578947368421052,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 15.802. Standardfehler: 2.5. Kosten je Aufgabe in USD: 0.006714. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 15.802. Standard error: 2.5. Cost per task in USD: 0.006714. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":5.194805194805195,"measurements":[{"observationId":"vals-lcb:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":37.088,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.194805194805195,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 5.349. Standardfehler: 1.091. Kosten je Aufgabe in USD: 0.003291. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 5.349. Standard error: 1.091. Cost per task in USD: 0.003291. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":9.090909090909092,"measurements":[{"observationId":"vals-math500:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":74.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.090909090909092,"participants":23,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 9.928. Standardfehler: 3.815. Kosten je Aufgabe in USD: 0.003323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 9.928. Standard error: 3.815. Cost per task in USD: 0.003323. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"vals-medqa:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":76.225,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.384615384615385,"participants":40,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 4.86. Standardfehler: 0.761. Kosten je Aufgabe in USD: 0.002428. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 4.86. Standard error: 0.761. Cost per task in USD: 0.002428. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":17.24137931034483,"measurements":[{"observationId":"vals-mgsm:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":87.236,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.24137931034483,"participants":30,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 8.041. Standardfehler: 2.045. Kosten je Aufgabe in USD: 0.002954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 8.041. Standard error: 2.045. Cost per task in USD: 0.002954. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":7.792207792207792,"measurements":[{"observationId":"vals-mmlu-pro:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":69.706,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.792207792207792,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 7.192. Standardfehler: 0.442. Kosten je Aufgabe in USD: 0.004997. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 7.192. Standard error: 0.442. Cost per task in USD: 0.004997. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":7.894736842105263,"measurements":[{"observationId":"vals-tax-eval-v2:overall:mistralai/mistral-large-2411","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":63.778,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.894736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-large-2411. Name des Quellmodells: Mistral Large 2.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 12.019. Standardfehler: 0.95. Kosten je Aufgabe in USD: 0.004079. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-large-2411. Source model name: Mistral Large 2.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 12.019. Standard error: 0.95. Cost per task in USD: 0.004079. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":7,"totalFamilies":71,"sourceCount":1,"coverage":0.09625850340136055,"lowerBound":1.066866436648651,"upperBound":91.4410160965126,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.3463203463203464,"upperBound":93.67965367965367,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":0.46992481203007513,"upperBound":93.32706766917293,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":5.2664576802507845,"upperBound":65.26645768025078,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":2.5974025974025974,"upperBound":69.26406926406926,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":1.1278195488721805,"upperBound":86.8421052631579,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":5.128205128205128,"upperBound":71.7948717948718,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-mistral-small-3-1-24b-instruct-2503","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-nemo","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-pixtral-12b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-pixtral-large-latest","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-small-3","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":1.4492753623188406,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:e5dd499f-c330-45ec-9ff0-a99209c82af7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":26.3945578231293,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.4492753623188406,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3 (Release mistral-small-3, Zeile mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/e5dd499f-c330-45ec-9ff0-a99209c82af7/mistral-small-3/mistral-small-3. Name des Quellmodells: Mistral Small 3. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 26,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3 (release mistral-small-3, row mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/e5dd499f-c330-45ec-9ff0-a99209c82af7/mistral-small-3/mistral-small-3. Source model name: Mistral Small 3. Provider reported by source: Mistral. Model snapshot: Mistral Small 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 26.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3 (Release mistral-small-3, Zeile mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3 (release mistral-small-3, row mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3 (Release mistral-small-3, Zeile mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3 (release mistral-small-3, row mistral-small-3, UUID e5dd499f-c330-45ec-9ff0-a99209c82af7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.02070393374741201,"upperBound":98.592132505176,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.2898550724637681,"upperBound":80.28985507246377,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-small-3.1","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":313.44,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":2.4390243902439024,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":22.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.4390243902439024,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 22,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 22.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.2009813466312218,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":377.03,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":7.246376811594203,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":29.9319727891156,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.246376811594203,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 29,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 29.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-50.75,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":3.289473684210526,"measurements":[{"observationId":"vals-gpqa:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":44.192,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.289473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 8.824. Standardfehler: 2.473. Kosten je Aufgabe in USD: 0.000278. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 8.824. Standard error: 2.473. Cost per task in USD: 0.000278. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":5.487804878048781,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.30954587581094,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":5.487804878048781,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":6.382978723404255,"measurements":[{"observationId":"vals-mmmu:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":60.081,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.382978723404255,"participants":48,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 13.401. Standardfehler: 1.177. Kosten je Aufgabe in USD: 0.000265. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 13.401. Standard error: 1.177. Cost per task in USD: 0.000265. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:24ac5b00-5f03-4c47-8e37-522d1195383e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/24ac5b00-5f03-4c47-8e37-522d1195383e/mistral-small-3-1/mistral-small-3-1. Source model name: Mistral Small 3.1. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.1 (Release mistral-small-3-1, Zeile mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.1 (release mistral-small-3-1, row mistral-small-3-1, UUID 24ac5b00-5f03-4c47-8e37-522d1195383e, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":1.3513513513513513,"measurements":[{"observationId":"vals-corp-fin-v2:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":44.173,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3513513513513513,"participants":75,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.7. Maximale Ausgabetoken: 131000. Latenz in Sekunden: 11.291. Standardfehler: 0.972. Kosten je Aufgabe in USD: 0.004408. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.7. Maximum output tokens: 131000. Latency in seconds: 11.291. Standard error: 0.972. Cost per task in USD: 0.004408. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 131000","en":"Temperature: 0.7. Maximum output tokens: 131000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 131000","en":"Temperature: 0.7. Maximum output tokens: 131000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":2.5974025974025974,"measurements":[{"observationId":"vals-lcb:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":31.815,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.5974025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 4.007. Standardfehler: 1.109. Kosten je Aufgabe in USD: 0.000147. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 4.007. Standard error: 1.109. Cost per task in USD: 0.000147. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":8.974358974358974,"measurements":[{"observationId":"vals-legal-bench:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":69.157,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.974358974358974,"participants":79,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 0.583. Standardfehler: 0.512. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 0.583. Standard error: 0.512. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"vals-math500:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":68.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":23,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 6.406. Standardfehler: 2.079. Kosten je Aufgabe in USD: 0.000158. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0. Maximum output tokens: 8192. Latency in seconds: 6.406. Standard error: 2.079. Cost per task in USD: 0.000158. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0. Maximale Ausgabetoken: 8192","en":"Temperature: 0. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":10.256410256410257,"measurements":[{"observationId":"vals-medqa:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":69.1,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.256410256410257,"participants":40,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 3.634. Standardfehler: 0.422. Kosten je Aufgabe in USD: 0.000102. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 3.634. Standard error: 0.422. Cost per task in USD: 0.000102. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":3.4482758620689653,"measurements":[{"observationId":"vals-mgsm:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":84.218,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.4482758620689653,"participants":30,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 3.87. Standardfehler: 2.21. Kosten je Aufgabe in USD: 0.000095. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 3.87. Standard error: 2.21. Cost per task in USD: 0.000095. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":3.896103896103896,"measurements":[{"observationId":"vals-mmlu-pro:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":66.021,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.896103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 3.602. Standardfehler: 0.455. Kosten je Aufgabe in USD: 0.000203. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 3.602. Standard error: 0.455. Cost per task in USD: 0.000203. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":19.23076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":61.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.23076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 8.313. Standardfehler: 0.956. Kosten je Aufgabe in USD: 0.000488. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 8.313. Standard error: 0.956. Cost per task in USD: 0.000488. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":1.3157894736842106,"measurements":[{"observationId":"vals-tax-eval-v2:overall:mistralai/mistral-small-2503","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":58.3,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3157894736842106,"participants":77,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2503. Name des Quellmodells: Mistral Small 3.1. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 8192. Latenz in Sekunden: 7.952. Standardfehler: 0.968. Kosten je Aufgabe in USD: 0.000184. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2503. Source model name: Mistral Small 3.1. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 8192. Latency in seconds: 7.952. Standard error: 0.968. Cost per task in USD: 0.000184. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 8192","en":"Temperature: 0.7. Maximum output tokens: 8192"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":20,"totalFamilies":71,"sourceCount":2,"coverage":0.227295918367347,"lowerBound":1.7151506623566748,"upperBound":78.98555882562198,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":0.7625910674691165,"upperBound":90.7625910674691,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":2.1948927196038874,"upperBound":59.337749862461024,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":0.689655172413793,"upperBound":60.689655172413794,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":6.826100728539753,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":4.550733819026502,"upperBound":37.88406715235984,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":4.776422764227642,"upperBound":79.77642276422765,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":1.4492753623188408,"upperBound":81.44927536231884,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":3.1282728651149703,"upperBound":60.27113000797211,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":2.2435897435897436,"upperBound":77.24358974358974,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":3.418803418803419,"upperBound":70.08547008547009,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":0.7978723404255319,"upperBound":88.29787234042553,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-small-3.2","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":3.048780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":3.048780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":1.2195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":20.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.2195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 20,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 20.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0.2854023922677291,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":1.829268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.829268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":1.2195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-194.91,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.2195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":13.043478260869565,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":33.469387755102,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.043478260869565,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 33,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 33.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-50.583333333333336,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":5.487804878048781,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":4.30954587581094,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":5.487804878048781,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 4,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 4.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":7.142857142857143,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":47.9768786127168,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.142857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning. Angezeigter Wert: 48,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning. Score display: 48.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:43da3718-3d6e-40dd-901a-05664179ff7f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Name des Quellmodells: Mistral Small 3.2. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 3.2. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/43da3718-3d6e-40dd-901a-05664179ff7f/mistral-small-3-2/mistral-small-3-2. Source model name: Mistral Small 3.2. Provider reported by source: Mistral. Model snapshot: Mistral Small 3.2. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 3.2 (Release mistral-small-3-2, Zeile mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Mistral Small 3.2 (release mistral-small-3-2, row mistral-small-3-2, UUID 43da3718-3d6e-40dd-901a-05664179ff7f, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.09277210884353743,"lowerBound":0.6743476818086728,"upperBound":91.39713679745493,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":0.8275261324041813,"upperBound":65.11324041811848,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":3.6585365853658542,"upperBound":70.32520325203252,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":0.8638211382113821,"upperBound":75.86382113821138,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":2.608695652173913,"upperBound":82.6086956521739,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":0.8928571428571429,"upperBound":88.39285714285714,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-small-4","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":1.25,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 1,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 1.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":241.11,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":49.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 49,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 49.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0.9635410038882158,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":1.829268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":1.829268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":362.6,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":26.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":-30.4,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":9.870250231696021,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 9,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 9.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":15.047909407665506,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:3fd96175-4ef1-434c-8795-f873aec2abc1","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral. Modellstand: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/3fd96175-4ef1-434c-8795-f873aec2abc1/mistral-small-4/mistral-small-4. Source model name: Mistral Small 4. Provider reported by source: Mistral. Model snapshot: Mistral Small 4 (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Mistral Small 4 (Reasoning) (Release mistral-small-4, Zeile mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Mistral Small 4 (Reasoning) (release mistral-small-4, row mistral-small-4, UUID 3fd96175-4ef1-434c-8795-f873aec2abc1, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:mistralai/mistral-small-2603","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":16.854,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":7.142857142857143,"participants":43,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2603. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 407.865. Standardfehler: 3.991. Kosten je Aufgabe in USD: 0.105762. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2603. Source model name: Mistral Small 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 407.865. Standard error: 3.991. Cost per task in USD: 0.105762. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":1.3888888888888888,"measurements":[{"observationId":"vals-vibe-code-bench:overall:mistralai/mistral-small-2603","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3888888888888888,"participants":73,"methodology":{"de":"Kennung des Quellmodells: mistralai/mistral-small-2603. Name des Quellmodells: Mistral Small 4. Anbieter laut Quelle: Mistral AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000. Latenz in Sekunden: 1814.681. Standardfehler: 0. Kosten je Aufgabe in USD: 6.473001. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: mistralai/mistral-small-2603. Source model name: Mistral Small 4. Provider reported by source: Mistral AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000. Latency in seconds: 1814.681. Standard error: 0. Cost per task in USD: 6.473001. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: high. Temperatur: 0.7. Maximale Ausgabetoken: 256000","en":"Harness: OpenHands. Reasoning effort: high. Temperature: 0.7. Maximum output tokens: 256000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":2,"coverage":0.08106575963718822,"lowerBound":1.2386390870711428,"upperBound":93.13206312335232,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.11111111111111113,"lowerBound":0.7613885662666152,"upperBound":89.6502774551555,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":3.5714285714285716,"upperBound":67.85714285714286,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":8.94308943089431,"upperBound":75.60975609756098,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":4.065040650406504,"upperBound":70.73170731707317,"provisional":true,"configurationsDiffer":true}}},{"modelId":"mistral-voxtral-mini-3b-2507","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mistral-voxtral-small-latest","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"mixtral-8x22b","contributions":[{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":5.128205128205129,"measurements":[{"observationId":"vals-medqa:overall:together/mistralai/Mixtral-8x22B-Instruct-v0.1","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":62.139,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.128205128205129,"participants":40,"methodology":{"de":"Kennung des Quellmodells: together/mistralai/Mixtral-8x22B-Instruct-v0.1. Name des Quellmodells: Mixtral 8x22B. Anbieter laut Quelle: Together AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 8.946. Standardfehler: 0.867. Kosten je Aufgabe in USD: 0.000918. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: together/mistralai/Mixtral-8x22B-Instruct-v0.1. Source model name: Mixtral 8x22B. Provider reported by source: Together AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Maximum output tokens: 4096. Latency in seconds: 8.946. Standard error: 0.867. Cost per task in USD: 0.000918. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Maximale Ausgabetoken: 4096","en":"Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Maximale Ausgabetoken: 4096","en":"Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.02380952380952381,"lowerBound":0.12210012210012212,"upperBound":97.74114774114774,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":1.7094017094017095,"upperBound":68.37606837606837,"provisional":true,"configurationsDiffer":true}}},{"modelId":"moonshotai-kimi-k2-thinking-turbo","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"moonshotai-kimi-k2.7-code-highspeed","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"motif-technologies-motif-3","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"naive-n0-5-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nemotron-3-nano","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nemotron-3-super","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nemotron-3-ultra","contributions":[{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":57.89473684210526,"measurements":[{"observationId":"vals-gpqa:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":86.112,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.89473684210526,"participants":77,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 158.471. Standardfehler: 1.738. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 158.471. Standard error: 1.738. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":19.491525423728813,"measurements":[{"observationId":"vals-swebench:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":69,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.491525423728813,"participants":60,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 283.089. Standardfehler: 2.07. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 283.089. Standard error: 2.07. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.16666666666666666,"score":23.214285714285715,"measurements":[{"observationId":"vals-terminal-bench-2-1:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":50.936,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.214285714285715,"participants":57,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 584.116. Standardfehler: 2.925. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 584.116. Standard error: 2.925. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":8.064516129032258,"measurements":[{"observationId":"vals-code-migration:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":4.905,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":8.064516129032258,"participants":63,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5470.513. Standardfehler: 1.589. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 5470.513. Standard error: 1.589. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":75.67567567567568,"measurements":[{"observationId":"vals-corp-fin-v2:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":65.462,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.67567567567568,"participants":75,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 15.015. Standardfehler: 0.937. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 15.015. Standard error: 0.937. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":16.949152542372882,"measurements":[{"observationId":"vals-emb:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":31.735,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.949152542372882,"participants":60,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2382.355. Standardfehler: 2.805. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2382.355. Standard error: 2.805. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":15,"measurements":[{"observationId":"vals-finance-agent-v2:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":37.667,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15,"participants":61,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 253.882. Standardfehler: 0.149. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 253.882. Standard error: 0.149. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":76.62337662337663,"measurements":[{"observationId":"vals-lcb:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":85.976,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.62337662337663,"participants":78,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 161.2. Standardfehler: 0.992. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 161.2. Standard error: 0.992. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":24.59016393442623,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.59016393442623,"participants":62,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2048.539. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2048.539. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":41.02564102564103,"measurements":[{"observationId":"vals-legal-bench:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.066,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.02564102564103,"participants":79,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5.179. Standardfehler: 0.46. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 5.179. Standard error: 0.46. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":19.166666666666668,"measurements":[{"observationId":"vals-legal-research:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":15.385,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":19.166666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 611.366. Standardfehler: 2.508. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 611.366. Standard error: 2.508. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":23.880597014925375,"measurements":[{"observationId":"vals-medcode:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":38.621,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":23.880597014925375,"participants":68,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 18.675. Standardfehler: 2.001. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 18.675. Standard error: 2.001. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":50.64935064935065,"measurements":[{"observationId":"vals-mmlu-pro:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.761,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.64935064935065,"participants":78,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 32.92. Standardfehler: 0.345. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 32.92. Standard error: 0.345. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3718.555. Standardfehler: 0. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 3718.555. Standard error: 0. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":53.94736842105263,"measurements":[{"observationId":"vals-tax-eval-v2:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":73.099,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.94736842105263,"participants":77,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 36.485. Standardfehler: 0.866. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 36.485. Standard error: 0.866. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":12.5,"measurements":[{"observationId":"vals-vibe-code-bench:overall:nvidia/nemotron-3-ultra-550b-a55b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":7.636,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.5,"participants":73,"methodology":{"de":"Kennung des Quellmodells: nvidia/nemotron-3-ultra-550b-a55b. Name des Quellmodells: Nemotron 3 Ultra. Anbieter laut Quelle: Nvidia. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2217.462. Standardfehler: 2.783. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: nvidia/nemotron-3-ultra-550b-a55b. Source model name: Nemotron 3 Ultra. Provider reported by source: Nvidia. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000. Latency in seconds: 2217.462. Standard error: 2.783. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Top P: 0.95. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Top P: 0.95. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":16,"totalFamilies":71,"sourceCount":1,"coverage":0.16933106575963722,"lowerBound":5.8768224607928925,"upperBound":88.94371588482917,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-15","areas":{"coding":{"score":null,"coveredFamilies":6,"totalFamilies":15,"sourceCount":1,"coverage":0.3111111111111112,"lowerBound":9.019499933445887,"upperBound":77.90838882233476,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":4.135338345864661,"upperBound":96.99248120300753,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.883116883116884,"upperBound":83.54978354978356,"provisional":true,"configurationsDiffer":true},"finance":{"score":40.3930491597753,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":23.081742377014457,"upperBound":65.93888523415731,"provisional":true,"configurationsDiffer":true},"legal":{"score":28.260823875577973,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":21.19561790668348,"upperBound":46.19561790668348,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":7.9601990049751254,"upperBound":74.6268656716418,"provisional":true,"configurationsDiffer":true}}},{"modelId":"nemotron-3.5-lightning","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":506.17,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":60.33333333333331,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 60,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 60.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.8410069634719541,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":3,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 3,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 3.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":638.57,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-17.716666666666665,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":10.5653382761817,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 10,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 10.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":5.084745762711864,"measurements":[{"observationId":"vals-swebench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":52.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":5.084745762711864,"participants":60,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 925.464. Standardfehler: 2.235. Kosten je Aufgabe in USD: 0.031537. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 925.464. Standard error: 2.235. Cost per task in USD: 0.031537. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":30.640243902439025,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:29976311-665a-4b2f-ac72-557c33e0758e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: NVIDIA. Modellstand: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/29976311-665a-4b2f-ac72-557c33e0758e/nemotron-3-5-lightning/nemotron-3-5-lightning. Source model name: Nemotron 3.5 Lightning. Provider reported by source: NVIDIA. Model snapshot: Nemotron 3.5 Lightning. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nemotron 3.5 Lightning (Release nemotron-3-5-lightning, Zeile nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Nemotron 3.5 Lightning (release nemotron-3-5-lightning, row nemotron-3-5-lightning, UUID 29976311-665a-4b2f-ac72-557c33e0758e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":10.861,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":57,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1028.327. Standardfehler: 2.622. Kosten je Aufgabe in USD: 0.014296. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1028.327. Standard error: 2.622. Cost per task in USD: 0.014296. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":3.225806451612903,"measurements":[{"observationId":"vals-code-migration:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":2.951,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.225806451612903,"participants":63,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2604.747. Standardfehler: 0.951. Kosten je Aufgabe in USD: 0.267017. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2604.747. Standard error: 0.951. Cost per task in USD: 0.267017. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":17.56756756756757,"measurements":[{"observationId":"vals-corp-fin-v2:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":54.235,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.56756756756757,"participants":75,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 38.792. Standardfehler: 0.98. Kosten je Aufgabe in USD: 0.004794. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 38.792. Standard error: 0.98. Cost per task in USD: 0.004794. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":3.0303030303030303,"measurements":[{"observationId":"vals-cyberbench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":29.464,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":3.0303030303030303,"participants":34,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 3031.784. Standardfehler: 5.958. Kosten je Aufgabe in USD: 0.093074. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 3031.784. Standard error: 5.958. Cost per task in USD: 0.093074. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":6.779661016949152,"measurements":[{"observationId":"vals-emb:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":16.987,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":6.779661016949152,"participants":60,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 2359.322. Standardfehler: 1.452. Kosten je Aufgabe in USD: 0.182153. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 2359.322. Standard error: 1.452. Cost per task in USD: 0.182153. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":0,"measurements":[{"observationId":"vals-finance-agent-v2:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":18.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":61,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 539.61. Standardfehler: 0.882. Kosten je Aufgabe in USD: 0.028554. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 539.61. Standard error: 0.882. Cost per task in USD: 0.028554. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1736.203. Standardfehler: 0. Kosten je Aufgabe in USD: 0.028287. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1736.203. Standard error: 0. Cost per task in USD: 0.028287. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":1.6666666666666667,"measurements":[{"observationId":"vals-legal-research:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":2.885,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.6666666666666667,"participants":61,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 285.48. Standardfehler: 1.163. Kosten je Aufgabe in USD: 0.004365. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 285.48. Standard error: 1.163. Cost per task in USD: 0.004365. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":0,"measurements":[{"observationId":"vals-medscribe:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":4.267,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":69,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 105.515. Standardfehler: 0.492. Kosten je Aufgabe in USD: 0.006241. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 105.515. Standard error: 0.492. Cost per task in USD: 0.006241. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1465.949. Standardfehler: 0. Kosten je Aufgabe in USD: 0.216901. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1465.949. Standard error: 0. Cost per task in USD: 0.216901. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":1.2820512820512822,"measurements":[{"observationId":"vals-proof-bench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.2820512820512822,"participants":40,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 919.248. Standardfehler: 0. Kosten je Aufgabe in USD: 0.015926. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 919.248. Standard error: 0. Cost per task in USD: 0.015926. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":1.8181818181818181,"measurements":[{"observationId":"vals-tax-agent-bench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":2.591,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.8181818181818181,"participants":56,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 1180.079. Standardfehler: 1.146. Kosten je Aufgabe in USD: 0.030533. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 1180.079. Standard error: 1.146. Cost per task in USD: 0.030533. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":1.3888888888888888,"measurements":[{"observationId":"vals-vibe-code-bench:overall:fireworks/nemotron-lightning-3p5-30b-a3b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":1.3888888888888888,"participants":73,"methodology":{"de":"Kennung des Quellmodells: fireworks/nemotron-lightning-3p5-30b-a3b. Name des Quellmodells: Nemotron 3.5 Lightning. Anbieter laut Quelle: Fireworks AI. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072. Latenz in Sekunden: 959.662. Standardfehler: 0. Kosten je Aufgabe in USD: 0.128799. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: fireworks/nemotron-lightning-3p5-30b-a3b. Source model name: Nemotron 3.5 Lightning. Provider reported by source: Fireworks AI. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072. Latency in seconds: 959.662. Standard error: 0. Cost per task in USD: 0.128799. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 131072","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 131072"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":21,"totalFamilies":71,"sourceCount":2,"coverage":0.2730725623582767,"lowerBound":2.765856409658586,"upperBound":75.45860017383092,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.2777777777777779,"lowerBound":3.4715795605678514,"upperBound":75.69380178279006,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":7.055749128919861,"upperBound":71.34146341463415,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0.25641025641025644,"upperBound":80.25641025641026,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":12.195121951219512,"upperBound":78.86178861788619,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":5.894308943089431,"upperBound":80.89430894308943,"provisional":true,"configurationsDiffer":true},"finance":{"score":6.541352600674634,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":3.7379157718140767,"upperBound":46.595058628956934,"provisional":true,"configurationsDiffer":true},"legal":{"score":6.16120218579235,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":3.080601092896175,"upperBound":53.080601092896174,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":0,"upperBound":66.66666666666667,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":3.0303030303030303,"upperBound":3.0303030303030303,"provisional":true,"configurationsDiffer":true}}},{"modelId":"nemotron-nano-12b-v2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nemotron-nano-9b-v2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nex-n2-5-max","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nex-n2-5-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nex-n2-5-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nex-n2-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nex-n2-pro","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":53.6231884057971,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:3b5ba264-ad25-429b-a460-4d8698205f0d","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=Based%20on%20Qwen3.5-397B-A17B","retrievedAt":"2026-10-19","value":66.1904761904762,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.6231884057971,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (Release nex-n2-pro, Zeile nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/3b5ba264-ad25-429b-a460-4d8698205f0d/nex-n2-pro/nex-n2-pro. Name des Quellmodells: Nex-N2-Pro. Anbieter laut Quelle: Nex AGI. Modellstand: Nex-N2-Pro (Based on Qwen3.5-397B-A17B). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Based on Qwen3.5-397B-A17B. Angezeigter Wert: 66,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (release nex-n2-pro, row nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/3b5ba264-ad25-429b-a460-4d8698205f0d/nex-n2-pro/nex-n2-pro. Source model name: Nex-N2-Pro. Provider reported by source: Nex AGI. Model snapshot: Nex-N2-Pro (Based on Qwen3.5-397B-A17B). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Based on Qwen3.5-397B-A17B. Score display: 66.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (Release nex-n2-pro, Zeile nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Based on Qwen3.5-397B-A17B","en":"Exact AA configuration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (release nex-n2-pro, row nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Based on Qwen3.5-397B-A17B"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (Release nex-n2-pro, Zeile nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: Based on Qwen3.5-397B-A17B","en":"Exact AA configuration: Nex-N2-Pro (Based on Qwen3.5-397B-A17B) (release nex-n2-pro, row nex-n2-pro, UUID 3b5ba264-ad25-429b-a460-4d8698205f0d, reasoning=true, source reasoning-token configuration budget=2000, suffix=Based on Qwen3.5-397B-A17B). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: Based on Qwen3.5-397B-A17B"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"Based on Qwen3.5-397B-A17B","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.7660455486542443,"upperBound":99.33747412008283,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":10.72463768115942,"upperBound":90.72463768115942,"provisional":true,"configurationsDiffer":true}}},{"modelId":"north-small-translate-1-0","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-llama-3.1-nemotron-70b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-llama-3.1-nemotron-ultra-253b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-llama-3.3-nemotron-super-49b-v1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-llama-3.3-nemotron-super-49b-v1.5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-mistral-nemotron","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-3-nano-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-3-nano-omni-30b-a3b-reasoning","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-3-super-120b-a12b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-3-ultra-550b-a55b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-3.5-lightning","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-cascade-2-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-mini-4b-instruct","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-nano-12b-v2-vl","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-nano-9b-v2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"nvidia-nemotron-voicechat","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"o1","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":68.1159420289855,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":70.3401360544218,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.1159420289855,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: o1 (Release o1, Zeile o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b/o1/o1. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 70,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o1 (release o1, row o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b/o1/o1. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 70.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o1 (Release o1, Zeile o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o1 (release o1, row o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o1 (Release o1, Zeile o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o1 (release o1, row o1, UUID 5ad2f60f-ee05-49fd-85a0-cef69aa7cb7b, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":22.36842105263158,"measurements":[{"observationId":"vals-gpqa:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":73.232,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":22.36842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 40.176. Standardfehler: 2.231. Kosten je Aufgabe in USD: 0.252632. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 40.176. Standard error: 2.231. Cost per task in USD: 0.252632. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-gaia-reliability","area":"agentic","label":{"de":"GAIA Reliability","en":"GAIA Reliability"},"weight":0.005952380952380952,"coverage":1,"score":12.5,"measurements":[{"observationId":"curated:hal-gaia-reliability-hal-reliability-overall-o1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/gaia/","benchmarkId":"hal-gaia-reliability","benchmarkVersion":"final leak-cleaned public validation set","comparisonCohort":"hal-gaia-reliability-hal-reliability-overall","retrievedAt":"2026-08-06","value":34.4,"unit":"%","metric":"exact-match accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":12.5,"participants":9,"methodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Kennung des Quellmodells: curated/o1. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-0b27954353a7. Prüfsumme der Quelldaten: 0b27954353a7c19a0cf4d633cf03db86774a4f1731e1998e0c8fb557f2d2a1e2. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Source model ID: curated/o1. Source model name: o1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-0b27954353a7. Source payload hash: 0b27954353a7c19a0cf4d633cf03db86774a4f1731e1998e0c8fb557f2d2a1e2. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation."},"scoreRunConfiguration":{"de":"165 GAIA-Aufgaben mit wiederholten Läufen und Perturbationen. Kosten veröffentlicht HAL für diese Auswertung nicht.. Harness: HAL Reliability. Agent: HAL Reliability","en":"165 GAIA tasks with repeated runs and perturbations. HAL does not publish cost for this evaluation.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-gaia-reliability-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"hal-tau-airline-clean","area":"agentic","label":{"de":"τ-bench Airline Clean","en":"τ-bench Airline Clean"},"weight":0.005952380952380952,"coverage":1,"score":25,"measurements":[{"observationId":"curated:hal-tau-airline-clean-hal-reliability-overall-o1","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/","benchmarkId":"hal-tau-airline-clean","benchmarkVersion":"26-task cleaned subset","comparisonCohort":"hal-tau-airline-clean-hal-reliability-overall","retrievedAt":"2026-08-05","value":66.2,"unit":"%","metric":"task success rate","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":25,"participants":9,"methodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Kennung des Quellmodells: curated/o1. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: HAL Reliability. Agent: HAL Reliability. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-5c14c4dac3c3. Prüfsumme der Quelldaten: 5c14c4dac3c3d877041cbb52e3bbb6b35af52ee20551afb0f3638c0c305ace9a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Source model ID: curated/o1. Source model name: o1. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: HAL Reliability. Agent: HAL Reliability. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-5c14c4dac3c3. Source payload hash: 5c14c4dac3c3d877041cbb52e3bbb6b35af52ee20551afb0f3638c0c305ace9a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset."},"scoreRunConfiguration":{"de":"26 bereinigte Airline-Aufgaben. Alle Kennzahlen wurden von Princeton HAL auf diesem Ausschnitt neu berechnet.. Harness: HAL Reliability. Agent: HAL Reliability","en":"26 cleaned airline tasks. Princeton HAL recomputed every metric from scratch on this subset.. Harness: HAL Reliability. Agent: HAL Reliability"},"scoreComparisonSetup":{"publicationCohort":"hal-tau-airline-clean-hal-reliability-overall","datasetType":"public","harness":"HAL Reliability","agent":"HAL Reliability","toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":false}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":21.27659574468085,"measurements":[{"observationId":"vals-mmmu:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":77.412,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.27659574468085,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 26.415. Standardfehler: 1.005. Kosten je Aufgabe in USD: 0.14586. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 26.415. Standard error: 1.005. Cost per task in USD: 0.14586. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":10.38961038961039,"measurements":[{"observationId":"vals-lcb:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":50.264,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.38961038961039,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 92.082. Standardfehler: 1.393. Kosten je Aufgabe in USD: 0.328067. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 92.082. Standard error: 1.393. Cost per task in USD: 0.328067. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":35.8974358974359,"measurements":[{"observationId":"vals-legal-bench:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":80.393,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.8974358974359,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 2048. Latenz in Sekunden: 8.338. Standardfehler: 0.578. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 2048. Latency in seconds: 8.338. Standard error: 0.578. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 2048","en":"Reasoning effort: high. Maximum output tokens: 2048"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 2048","en":"Reasoning effort: high. Maximum output tokens: 2048"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":50,"measurements":[{"observationId":"vals-math500:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":90.4,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 23.537. Standardfehler: 2.591. Kosten je Aufgabe in USD: 0.095269. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 32768. Latency in seconds: 23.537. Standard error: 2.591. Cost per task in USD: 0.095269. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":100,"measurements":[{"observationId":"vals-medqa:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.517,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":100,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 11.152. Standardfehler: 0.33. Kosten je Aufgabe in USD: 0.05282. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Maximum output tokens: 4096. Latency in seconds: 11.152. Standard error: 0.33. Cost per task in USD: 0.05282. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":43.10344827586207,"measurements":[{"observationId":"vals-mgsm:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":89.309,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.10344827586207,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 11.207. Standardfehler: 1.779. Kosten je Aufgabe in USD: 0.057248. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 11.207. Standard error: 1.779. Cost per task in USD: 0.057248. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":40.25974025974026,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":83.488,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.25974025974026,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 26.867. Standardfehler: 0.364. Kosten je Aufgabe in USD: 0.116223. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 26.867. Standard error: 0.364. Cost per task in USD: 0.116223. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":64.47368421052632,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/o1-2024-12-17","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.284,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.47368421052632,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o1-2024-12-17. Name des Quellmodells: o1. Anbieter laut Quelle: OpenAI. Modellstand: o1-2024-12-17. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 19.477. Standardfehler: 0.861. Kosten je Aufgabe in USD: 0.12173. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o1-2024-12-17. Source model name: o1. Provider reported by source: OpenAI. Model snapshot: o1-2024-12-17. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 19.477. Standard error: 0.861. Cost per task in USD: 0.12173. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":12,"totalFamilies":71,"sourceCount":3,"coverage":0.14923469387755106,"lowerBound":7.518354973285198,"upperBound":92.5948855855301,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.0666666666666667,"lowerBound":0.6926406926406928,"upperBound":94.02597402597402,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.5977443609022555,"upperBound":94.45488721804512,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":18.620689655172413,"upperBound":78.62068965517241,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":13.41991341991342,"upperBound":80.08658008658008,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":3.125,"upperBound":86.45833333333334,"provisional":true,"configurationsDiffer":false},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":13.623188405797102,"upperBound":93.6231884057971,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":9.210526315789473,"upperBound":94.9248120300752,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":8.974358974358974,"upperBound":83.97435897435898,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":33.333333333333336,"upperBound":100,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":2.6595744680851063,"upperBound":90.15957446808511,"provisional":true,"configurationsDiffer":true}}},{"modelId":"o3","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":72.46376811594203,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:16c5b637-8bce-4252-81f2-1b87a36a4e4c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":71.4285714285714,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":72.46376811594203,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/16c5b637-8bce-4252-81f2-1b87a36a4e4c/o3/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 71,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/16c5b637-8bce-4252-81f2-1b87a36a4e4c/o3/o3. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 71.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":49.3421052631579,"measurements":[{"observationId":"vals-gpqa:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":84.091,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.3421052631579,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 65.777. Standardfehler: 1.861. Kosten je Aufgabe in USD: 0.035552. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 65.777. Standard error: 1.861. Cost per task in USD: 0.035552. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-assistantbench","area":"agentic","label":{"de":"AssistantBench","en":"AssistantBench"},"weight":0.005952380952380952,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-assistantbench-browser-use-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/assistantbench","benchmarkId":"hal-assistantbench","benchmarkVersion":"HAL 33-task public set","comparisonCohort":"hal-assistantbench-browser-use-overall","retrievedAt":"2026-08-05","value":38.81,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":5,"methodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 0.4590909090909091. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-841883e41588. Prüfsumme der Quelldaten: 841883e4158871aba9a9d77489fe6a278e31e51c19da95a103f354da8eef2ce1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium. Cost per task in USD: 0.4590909090909091. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-841883e41588. Source payload hash: 841883e4158871aba9a9d77489fe6a278e31e51c19da95a103f354da8eef2ce1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33."},"scoreRunConfiguration":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-assistantbench-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":40,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":29,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":40,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 1.2386333333333333. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-77ccfede6f14. Prüfsumme der Quelldaten: 77ccfede6f14eea25d1a9500c263a7724e8796b6e0eadad09ca22ede439c8eee. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium. Cost per task in USD: 1.2386333333333333. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-77ccfede6f14. Source payload hash: 77ccfede6f14eea25d1a9500c263a7724e8796b6e0eadad09ca22ede439c8eee. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: medium","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":33.33,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 0.1146078431372549. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-9ade5dc68557. Prüfsumme der Quelldaten: 9ade5dc685576c1832cf9d48efadbc1874421ff2f41acf31b26b010ad5d90e6c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: medium. Cost per task in USD: 0.1146078431372549. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-9ade5dc68557. Source payload hash: 9ade5dc685576c1832cf9d48efadbc1874421ff2f41acf31b26b010ad5d90e6c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: medium","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":60,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":46.25,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":60,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 0.18664495114006513. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-a6c14f678280. Prüfsumme der Quelldaten: a6c14f67828082f2e362b304d2937f0008fbe7325b9f9537343e362916bda5c2. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: medium. Cost per task in USD: 0.18664495114006513. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-a6c14f678280. Source payload hash: a6c14f67828082f2e362b304d2937f0008fbe7325b9f9537343e362916bda5c2. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: medium","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":33.53913373860182,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:16c5b637-8bce-4252-81f2-1b87a36a4e4c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":70.0578034682081,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.035714285714285,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/16c5b637-8bce-4252-81f2-1b87a36a4e4c/o3/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 70,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/16c5b637-8bce-4252-81f2-1b87a36a4e4c/o3/o3. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 70.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3 (Release o3, Zeile o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: o3 (release o3, row o3, UUID 16c5b637-8bce-4252-81f2-1b87a36a4e4c, reasoning=true, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":80.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.04255319148936,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 39.275. Standardfehler: 0.954. Kosten je Aufgabe in USD: 0.015444. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 39.275. Standard error: 0.954. Cost per task in USD: 0.015444. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":28.571428571428573,"measurements":[{"observationId":"curated:scale-multichallenge-o3","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":56.62,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":28.571428571428573,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 56,62 % ± 3,32. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-331c80814a4d. Prüfsumme der Quelldaten: 331c80814a4d56cbdc0a8d3bbb1be4831a0c1bf8683c511b680a1e30998312c0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 56.62% ± 3.32. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-331c80814a4d. Source payload hash: 331c80814a4d56cbdc0a8d3bbb1be4831a0c1bf8683c511b680a1e30998312c0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: high","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":36.36363636363637,"measurements":[{"observationId":"curated:scale-multinrc-o3-high","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":45.5,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":36.36363636363637,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 45,5 % ± 3. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-4aa6d654620e. Prüfsumme der Quelldaten: 4aa6d654620e605fb9b1f052c7367d1ec8b478bec040e5fbc0587fd3ae14ad60. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 45.5% ± 3. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-4aa6d654620e. Source payload hash: 4aa6d654620e605fb9b1f052c7367d1ec8b478bec040e5fbc0587fd3ae14ad60. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1. Reasoning-Stufe: high","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":60,"measurements":[{"observationId":"curated:scale-prbench-finance-o3","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":47.69,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":60,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 47,69 % ± 0,45. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-882710175a98. Prüfsumme der Quelldaten: 882710175a983d59d9f697f4f752fb907874fd124f6b244a4c8726b90f29b9a1. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 47.69% ± 0.45. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-882710175a98. Source payload hash: 882710175a983d59d9f697f4f752fb907874fd124f6b244a4c8726b90f29b9a1. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":60,"measurements":[{"observationId":"curated:scale-prbench-legal-o3","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":48.57,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":60,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 48,57 % ± 0,54. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-9515f9532787. Prüfsumme der Quelldaten: 9515f953278706b5d469c0fcb47a80c2b9a900bb162fa270b397c4a49bad8301. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 48.57% ± 0.54. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-9515f9532787. Source payload hash: 9515f953278706b5d469c0fcb47a80c2b9a900bb162fa270b397c4a49bad8301. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"scicode","area":"coding","label":{"de":"SciCode","en":"SciCode"},"weight":0.004761904761904762,"coverage":1,"score":100,"measurements":[{"observationId":"curated:hal-scicode-scicode-tool-calling-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scicode","benchmarkId":"hal-scicode","benchmarkVersion":"65-problem main set","comparisonCohort":"hal-scicode-scicode-tool-calling-overall","retrievedAt":"2026-08-05","value":9.23,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":5,"methodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 1.7093846153846153. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-3b73275854bb. Prüfsumme der Quelldaten: 3b73275854bb2de797dd9ed67237c8ad83c065e9283768a925ef571bd04c9156. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: medium. Cost per task in USD: 1.7093846153846153. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-3b73275854bb. Source payload hash: 3b73275854bb2de797dd9ed67237c8ad83c065e9283768a925ef571bd04c9156. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65."},"scoreRunConfiguration":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: medium","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-scicode-scicode-tool-calling-overall","datasetType":"public","harness":"SciCode Tool Calling","agent":"SciCode Tool Calling","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":62.5,"measurements":[{"observationId":"curated:hal-swebench-verified-mini-swe-agent-overall-o3","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/swebench_verified_mini","benchmarkId":"hal-swebench-verified-mini","benchmarkVersion":"50-task random verified subset","comparisonCohort":"hal-swebench-verified-mini-swe-agent-overall","retrievedAt":"2026-08-05","value":46,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":62.5,"participants":5,"methodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: medium. Kosten je Aufgabe in USD: 9.6686. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-dfc1141bc4df. Prüfsumme der Quelldaten: dfc1141bc4df34f7ad53b05ff9088f1baa14b509e1ab3d6f61197895235f081c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: medium. Cost per task in USD: 9.6686. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-dfc1141bc4df. Source payload hash: dfc1141bc4df34f7ad53b05ff9088f1baa14b509e1ab3d6f61197895235f081c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50."},"scoreRunConfiguration":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: medium","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: medium"},"scoreComparisonSetup":{"publicationCohort":"hal-swebench-verified-mini-swe-agent-overall","datasetType":"public","harness":"SWE-Agent","agent":"SWE-Agent","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"medium","weight":0.5,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":33.333333333333336,"measurements":[{"observationId":"curated:scale-tutorbench-o3-high","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":52.09,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":33.333333333333336,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 52,09 % ± 1,01. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-4708fc7ef46d. Prüfsumme der Quelldaten: 4708fc7ef46df8002e97262532cbfa7214ddd9fcd9dba7aebcc4ccb9db54195f. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 52.09% ± 1.01. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-4708fc7ef46d. Source payload hash: 4708fc7ef46df8002e97262532cbfa7214ddd9fcd9dba7aebcc4ccb9db54195f. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1. Reasoning-Stufe: high","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":32.432432432432435,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":59.713,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.432432432432435,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 19.153. Standardfehler: 0.965. Kosten je Aufgabe in USD: 0.133487. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 19.153. Standard error: 0.965. Cost per task in USD: 0.133487. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":58.44155844155844,"measurements":[{"observationId":"vals-lcb:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":83.914,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.44155844155844,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 63.948. Standardfehler: 1.03. Kosten je Aufgabe in USD: 0.032963. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 63.948. Standard error: 1.03. Cost per task in USD: 0.032963. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":57.69230769230769,"measurements":[{"observationId":"vals-legal-bench:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.761,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":57.69230769230769,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 5.979. Standardfehler: 0.424. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 5.979. Standard error: 0.424. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":86.36363636363636,"measurements":[{"observationId":"vals-math500:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.36363636363636,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 16.591. Standardfehler: 1.011. Kosten je Aufgabe in USD: 0.010795. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 16.591. Standard error: 1.011. Cost per task in USD: 0.010795. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":62.6865671641791,"measurements":[{"observationId":"vals-medcode:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":47.29,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.6865671641791,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 17.676. Standardfehler: 2.161. Kosten je Aufgabe in USD: 0.029818. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 17.676. Standard error: 2.161. Cost per task in USD: 0.029818. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":88.46153846153847,"measurements":[{"observationId":"vals-medqa:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.058,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.46153846153847,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 8.507. Standardfehler: 0.178. Kosten je Aufgabe in USD: 0.00542. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 8.507. Standard error: 0.178. Cost per task in USD: 0.00542. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":30.88235294117647,"measurements":[{"observationId":"vals-medscribe:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":76.654,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30.88235294117647,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 46.392. Standardfehler: 1.871. Kosten je Aufgabe in USD: 0.040334. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 46.392. Standard error: 1.871. Cost per task in USD: 0.040334. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":72.41379310344827,"measurements":[{"observationId":"vals-mgsm:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":91.746,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":72.41379310344827,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 6.76. Standardfehler: 1.673. Kosten je Aufgabe in USD: 0.003754. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 6.76. Standard error: 1.673. Cost per task in USD: 0.003754. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":49.35064935064935,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":85.595,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":49.35064935064935,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 16.735. Standardfehler: 0.345. Kosten je Aufgabe in USD: 0.011827. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 16.735. Standard error: 0.345. Cost per task in USD: 0.011827. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":44.23076923076923,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":65.7,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.23076923076923,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 21.216. Standardfehler: 0.933. Kosten je Aufgabe in USD: 0.009972. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 21.216. Standard error: 0.933. Cost per task in USD: 0.009972. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":29.310344827586206,"measurements":[{"observationId":"vals-sage:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":41.771,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.310344827586206,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 48.024. Standardfehler: 3.315. Kosten je Aufgabe in USD: 0.027758. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 48.024. Standard error: 3.315. Cost per task in USD: 0.027758. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":69.73684210526316,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/o3-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.571,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.73684210526316,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o3-2025-04-16. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Modellstand: o3-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 24.024. Standardfehler: 0.854. Kosten je Aufgabe in USD: 0.012997. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-2025-04-16. Source model name: o3. Provider reported by source: OpenAI. Model snapshot: o3-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 24.024. Standard error: 0.854. Cost per task in USD: 0.012997. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":46.15384615384615,"measurements":[{"observationId":"curated:scale-vista-o3-high","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":50.07,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":46.15384615384615,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 50,07 % ± 1,14. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-faa4d7636f7a. Prüfsumme der Quelldaten: faa4d7636f7ad0415c7e6ea2ef5cbef47f2b10b7b21b1902de2ad5a4e69b3075. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 50.07% ± 1.14. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-faa4d7636f7a. Source payload hash: faa4d7636f7ad0415c7e6ea2ef5cbef47f2b10b7b21b1902de2ad5a4e69b3075. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1. Reasoning-Stufe: high","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"curated:scale-visual-toolbench-o3","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":13.74,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/o3. Name des Quellmodells: o3. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 13,74 % ± 1,78. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-8b40533e4cb9. Prüfsumme der Quelldaten: 8b40533e4cb9ff0ac41bc5525aeb5894e5c8004455b0457dfcbba15a822b2a01. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/o3. Source model name: o3. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 13.74% ± 1.78. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-8b40533e4cb9. Source payload hash: 8b40533e4cb9ff0ac41bc5525aeb5894e5c8004455b0457dfcbba15a822b2a01. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":28,"totalFamilies":71,"sourceCount":4,"coverage":0.4239795918367348,"lowerBound":22.01287260506926,"upperBound":79.61491342139577,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.3000000000000001,"lowerBound":23.312770562770567,"upperBound":93.31277056277057,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":2,"coverage":0.2142857142857143,"lowerBound":8.71924128503076,"upperBound":87.29066985645933,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":31.755485893416925,"upperBound":91.75548589341693,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":16.45021645021645,"upperBound":83.11688311688312,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":11.666666666666666,"upperBound":95.00000000000001,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":14.492753623188406,"upperBound":94.4927536231884,"provisional":true,"configurationsDiffer":true},"finance":{"score":51.600010942116214,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":29.485720538352123,"upperBound":72.34286339549499,"provisional":true,"configurationsDiffer":true},"legal":{"score":58.84615384615384,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.5,"lowerBound":29.42307692307692,"upperBound":79.42307692307692,"provisional":true,"configurationsDiffer":true},"health":{"score":60.67681952229801,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":60.67681952229801,"upperBound":60.67681952229801,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":48.78654885303821,"coveredFamilies":3,"totalFamilies":4,"sourceCount":3,"coverage":0.75,"lowerBound":36.58991163977866,"upperBound":61.58991163977866,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":14.285714285714286,"upperBound":64.28571428571429,"provisional":true,"configurationsDiffer":true},"education":{"score":31.32183908045977,"coveredFamilies":2,"totalFamilies":2,"sourceCount":2,"coverage":1,"lowerBound":31.32183908045977,"upperBound":31.32183908045977,"provisional":true,"configurationsDiffer":true}}},{"modelId":"o3-mini","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":141.71,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":43,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 43,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 43.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":9.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":0.5301828617801994,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":9.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":16.463414634146343,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":16.463414634146343,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 2,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 2.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":230.33,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":55.79710144927536,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":67.14285714285711,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.79710144927536,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 67,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 67.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":-42.56666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":26.31578947368421,"measurements":[{"observationId":"vals-gpqa:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":75.505,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.31578947368421,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 99.984. Standardfehler: 2.176. Kosten je Aufgabe in USD: 0.035857. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 99.984. Standard error: 2.176. Cost per task in USD: 0.035857. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":12.04,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 12,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 12.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:076f2674-bc4b-4925-be59-50832eb8c090","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Source model ID: artificial-analysis/076f2674-bc4b-4925-be59-50832eb8c090/o3-mini/o3-mini-high. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o3-mini (High) (Release o3-mini, Zeile o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o3-mini (High) (release o3-mini, row o3-mini-high, UUID 076f2674-bc4b-4925-be59-50832eb8c090, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":2.7027027027027026,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":45.299,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.7027027027027026,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 31.047. Standardfehler: 0.964. Kosten je Aufgabe in USD: 0.078121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 31.047. Standard error: 0.964. Cost per task in USD: 0.078121. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":25.974025974025974,"measurements":[{"observationId":"vals-lcb:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":71.484,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.974025974025974,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 53.8. Standardfehler: 1.154. Kosten je Aufgabe in USD: 0.033121. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 53.8. Standard error: 1.154. Cost per task in USD: 0.033121. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":11.538461538461538,"measurements":[{"observationId":"vals-legal-bench:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":71.539,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":11.538461538461538,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 12.011. Standardfehler: 0.72. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 4096. Latency in seconds: 12.011. Standard error: 0.72. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":54.54545454545455,"measurements":[{"observationId":"vals-math500:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":91.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":54.54545454545455,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 32768. Latenz in Sekunden: 14.365. Standardfehler: 2.417. Kosten je Aufgabe in USD: 0.007661. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 32768. Latency in seconds: 14.365. Standard error: 2.417. Cost per task in USD: 0.007661. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Maximum output tokens: 32768"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 32768","en":"Reasoning effort: high. Maximum output tokens: 32768"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":79.48717948717949,"measurements":[{"observationId":"vals-medqa:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":94.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.48717948717949,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Maximale Ausgabetoken: 4096. Latenz in Sekunden: 8.387. Standardfehler: 0.397. Kosten je Aufgabe in USD: 0.005327. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Maximum output tokens: 4096. Latency in seconds: 8.387. Standard error: 0.397. Cost per task in USD: 0.005327. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 4096","en":"Reasoning effort: high. Maximum output tokens: 4096"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":65.51724137931035,"measurements":[{"observationId":"vals-mgsm:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":91.346,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":65.51724137931035,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 15.976. Standardfehler: 1.718. Kosten je Aufgabe in USD: 0.008194. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 15.976. Standard error: 1.718. Cost per task in USD: 0.008194. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":16.883116883116884,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":78.689,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.883116883116884,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 22.375. Standardfehler: 0.394. Kosten je Aufgabe in USD: 0.012991. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 22.375. Standard error: 0.394. Cost per task in USD: 0.012991. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":21.05263157894737,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/o3-mini-2025-01-31","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":69.42,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.05263157894737,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o3-mini-2025-01-31. Name des Quellmodells: o3-mini. Anbieter laut Quelle: OpenAI. Modellstand: o3-mini-2025-01-31. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 74.312. Standardfehler: 0.909. Kosten je Aufgabe in USD: 0.018142. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o3-mini-2025-01-31. Source model name: o3-mini. Provider reported by source: OpenAI. Model snapshot: o3-mini-2025-01-31. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 74.312. Standard error: 0.909. Cost per task in USD: 0.018142. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":18,"totalFamilies":71,"sourceCount":2,"coverage":0.2081632653061225,"lowerBound":6.5321654448666235,"upperBound":85.71583891425438,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":2,"totalFamilies":15,"sourceCount":2,"coverage":0.10000000000000003,"lowerBound":2.3210326259106755,"upperBound":92.32103262591066,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":7.628828167980927,"upperBound":64.77168531083807,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":24.01253918495298,"upperBound":84.01253918495298,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":16.368387709851124,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":10.912258473234083,"upperBound":44.24559180656742,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":2.6422764227642275,"upperBound":77.64227642276423,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.159420289855072,"upperBound":91.15942028985508,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":2,"totalFamilies":7,"sourceCount":1,"coverage":0.2857142857142857,"lowerBound":3.3936191830928677,"upperBound":74.8221906116643,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":2.8846153846153846,"upperBound":77.88461538461539,"provisional":true,"configurationsDiffer":true},"health":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":26.495726495726498,"upperBound":93.16239316239317,"provisional":true,"configurationsDiffer":true}}},{"modelId":"o3-pro","contributions":[{"familyId":"multichallenge","area":"productivity","label":{"de":"MultiChallenge","en":"MultiChallenge"},"weight":0.03571428571428571,"coverage":1,"score":57.142857142857146,"measurements":[{"observationId":"curated:scale-multichallenge-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multichallenge","benchmarkId":"multichallenge","benchmarkVersion":"four multi-turn challenge types","comparisonCohort":"scale-multichallenge-overall","retrievedAt":"2026-08-26","value":62.4,"unit":"%","metric":"rubric pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":57.142857142857146,"participants":8,"methodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 62,4 % ± 2,49. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-658a3e008a02. Prüfsumme der Quelldaten: 658a3e008a023110d3ccb0ba0342e3d55eb690fea0e546fd5c644a1287842ce7. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Multi-turn conversations with binary task-specific rubrics.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 62.4% ± 2.49. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-658a3e008a02. Source payload hash: 658a3e008a023110d3ccb0ba0342e3d55eb690fea0e546fd5c644a1287842ce7. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.","en":"Multi-turn conversations with binary task-specific rubrics."},"scoreRunConfiguration":{"de":"Mehrstufige Gespräche mit binären, aufgabenspezifischen Rubriken.. Versuche: pass@1. Reasoning-Stufe: high","en":"Multi-turn conversations with binary task-specific rubrics.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-multichallenge-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"multinrc","area":"reasoning","label":{"de":"MultiNRC","en":"MultiNRC"},"weight":0.01020408163265306,"coverage":1,"score":45.45454545454545,"measurements":[{"observationId":"curated:scale-multinrc-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/multinrc","benchmarkId":"multinrc","benchmarkVersion":"French, Spanish, and Chinese release","comparisonCohort":"scale-multinrc-native-languages","retrievedAt":"2026-08-26","value":49,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":45.45454545454545,"participants":12,"methodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 49 % ± 3,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-145f9b03fae8. Prüfsumme der Quelldaten: 145f9b03fae81a57d6da422e17ffd143e4dd7869da95b1408b0f8ce896b4246a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"Native, non-translated reasoning tasks with automatic evaluation.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 49% ± 3.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-145f9b03fae8. Source payload hash: 145f9b03fae81a57d6da422e17ffd143e4dd7869da95b1408b0f8ce896b4246a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.","en":"Native, non-translated reasoning tasks with automatic evaluation."},"scoreRunConfiguration":{"de":"Native, nicht aus dem Englischen übersetzte Reasoning-Aufgaben mit automatischer Bewertung.. Versuche: pass@1. Reasoning-Stufe: high","en":"Native, non-translated reasoning tasks with automatic evaluation.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-multinrc-native-languages","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"curated:scale-prbench-finance-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":49.08,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 49,08 % ± 0,79. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-a15dd5860124. Prüfsumme der Quelldaten: a15dd5860124c5d2805f234fc9e99880171cd19f76f66cacf63f4ab21a846ef0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 49.08% ± 0.79. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-a15dd5860124. Source payload hash: a15dd5860124c5d2805f234fc9e99880171cd19f76f66cacf63f4ab21a846ef0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":73.33333333333333,"measurements":[{"observationId":"curated:scale-prbench-legal-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":49.67,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":73.33333333333333,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 49,67 % ± 0,5. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-e1dcf6ae7d33. Prüfsumme der Quelldaten: e1dcf6ae7d33b234307f8bd4ba4bb133394e4cbca8c6ada0a92a5d927367d961. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 49.67% ± 0.5. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-e1dcf6ae7d33. Source payload hash: e1dcf6ae7d33b234307f8bd4ba4bb133394e4cbca8c6ada0a92a5d927367d961. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"tutorbench","area":"education","label":{"de":"TutorBench","en":"TutorBench"},"weight":0.03571428571428571,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"curated:scale-tutorbench-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/tutorbench","benchmarkId":"tutorbench","benchmarkVersion":"1,490-task release","comparisonCohort":"scale-tutorbench-overall","retrievedAt":"2026-08-26","value":54.62,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":66.66666666666667,"participants":10,"methodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 54,62 % ± 1,02. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-7a86c0e52382. Prüfsumme der Quelldaten: 7a86c0e52382f7157e90b4a75ffca0e79c104460ae4ac971206896a283b99856. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 54.62% ± 1.02. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-7a86c0e52382. Source payload hash: 7a86c0e52382f7157e90b4a75ffca0e79c104460ae4ac971206896a283b99856. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge."},"scoreRunConfiguration":{"de":"1.490 Aufgaben mit gewichteten Rubriken und Claude 4 Sonnet als Juror.. Versuche: pass@1","en":"1,490 tasks with weighted rubrics and Claude 4 Sonnet as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-tutorbench-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":69.23076923076923,"measurements":[{"observationId":"curated:scale-vista-o3-pro","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":51.63,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":69.23076923076923,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/o3-pro. Name des Quellmodells: o3-pro. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 51,63 % ± 0,25. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-8a92ee7cf963. Prüfsumme der Quelldaten: 8a92ee7cf963b85d0a28682e186b6062fef61e9317b71ec8603f644cdb8020fd. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/o3-pro. Source model name: o3-pro. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 51.63% ± 0.25. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-8a92ee7cf963. Source payload hash: 8a92ee7cf963b85d0a28682e186b6062fef61e9317b71ec8603f644cdb8020fd. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1. Reasoning-Stufe: high","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":6,"totalFamilies":71,"sourceCount":1,"coverage":0.1275510204081633,"lowerBound":8.111650254507397,"upperBound":95.35654821369106,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-26","retrievedThrough":"2026-08-26","areas":{"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":6.493506493506493,"upperBound":92.20779220779221,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":9.523809523809524,"upperBound":95.23809523809524,"provisional":true,"configurationsDiffer":true},"legal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":18.333333333333332,"upperBound":93.33333333333333,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.25,"lowerBound":17.307692307692307,"upperBound":92.3076923076923,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":28.571428571428573,"upperBound":78.57142857142857,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":33.333333333333336,"upperBound":83.33333333333334,"provisional":true,"configurationsDiffer":true}}},{"modelId":"o4-mini","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":62.31884057971015,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:84b49308-6b93-47aa-a4f6-776ee1a1e8cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":68.7074829931973,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.31884057971015,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/84b49308-6b93-47aa-a4f6-776ee1a1e8cd/o4-mini/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 68,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/84b49308-6b93-47aa-a4f6-776ee1a1e8cd/o4-mini/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 68.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":"High","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":24.342105263157894,"measurements":[{"observationId":"vals-gpqa:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":74.495,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.342105263157894,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 30.004. Standardfehler: 2.19. Kosten je Aufgabe in USD: 0.012961. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 30.004. Standard error: 2.19. Cost per task in USD: 0.012961. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"hal-assistantbench","area":"agentic","label":{"de":"AssistantBench","en":"AssistantBench"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"curated:hal-assistantbench-browser-use-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/assistantbench","benchmarkId":"hal-assistantbench","benchmarkVersion":"HAL 33-task public set","comparisonCohort":"hal-assistantbench-browser-use-overall","retrievedAt":"2026-08-05","value":23.84,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":50,"participants":5,"methodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 0.4966666666666667. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-3d1f814e517d. Prüfsumme der Quelldaten: 3d1f814e517d5fcf43880850242ddaf523f7934d4a592c937fd063d913ad21c7. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: high. Cost per task in USD: 0.4966666666666667. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-3d1f814e517d. Source payload hash: 3d1f814e517d5fcf43880850242ddaf523f7934d4a592c937fd063d913ad21c7. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33."},"scoreRunConfiguration":{"de":"Ein gemeinsames Browser-Use-Gerüst löst 33 verifizierte HAL-Aufgaben. Kosten sind durch 33 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: high","en":"One shared Browser-Use scaffold solves 33 verified HAL tasks. Total cost is divided by 33.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-assistantbench-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-online-mind2web","area":"agentic","label":{"de":"Online Mind2Web","en":"Online Mind2Web"},"weight":0.005952380952380952,"coverage":1,"score":0,"measurements":[{"observationId":"curated:hal-online-mind2web-browser-use-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/online_mind2web","benchmarkId":"hal-online-mind2web","benchmarkVersion":"300-task live set","comparisonCohort":"hal-online-mind2web-browser-use-overall","retrievedAt":"2026-08-05","value":20,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":0,"participants":6,"methodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 0.9931. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-f493207d2ef4. Prüfsumme der Quelldaten: f493207d2ef482a5dc6e2b10f3497245cec438d7f78bf728280b30ad90f1990e. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: high. Cost per task in USD: 0.9931. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-f493207d2ef4. Source payload hash: f493207d2ef482a5dc6e2b10f3497245cec438d7f78bf728280b30ad90f1990e. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300."},"scoreRunConfiguration":{"de":"Das Browser-Use-Gerüst bearbeitet dieselben 300 Aufgaben auf 136 Live-Websites. Kosten sind durch 300 geteilt.. Harness: Browser-Use. Agent: Browser-Use. Reasoning-Stufe: high","en":"The Browser-Use scaffold handles the same 300 tasks across 136 live websites. Total cost is divided by 300.. Harness: Browser-Use. Agent: Browser-Use. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-online-mind2web-browser-use-overall","datasetType":"public","harness":"Browser-Use","agent":"Browser-Use","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-scienceagentbench","area":"coding","label":{"de":"ScienceAgentBench","en":"ScienceAgentBench"},"weight":0.004761904761904762,"coverage":1,"score":60,"measurements":[{"observationId":"curated:hal-scienceagentbench-sab-self-debug-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scienceagentbench","benchmarkId":"hal-scienceagentbench","benchmarkVersion":"verified 102-task set","comparisonCohort":"hal-scienceagentbench-sab-self-debug-overall","retrievedAt":"2026-08-06","value":27.45,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":60,"participants":6,"methodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 0.1096078431372549. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-0b1cf3a90503. Prüfsumme der Quelldaten: 0b1cf3a90503438e0729d0f9a80efd444f1e44876bb0d54b94fde82de401c3ef. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: high. Cost per task in USD: 0.1096078431372549. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-0b1cf3a90503. Source payload hash: 0b1cf3a90503438e0729d0f9a80efd444f1e44876bb0d54b94fde82de401c3ef. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL."},"scoreRunConfiguration":{"de":"SAB Self-Debug auf 102 wissenschaftlichen Aufgaben. Kosten sind durch 102 geteilt und berücksichtigen laut HAL keinen Cache.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning-Stufe: high","en":"SAB Self-Debug on 102 scientific tasks. Total cost is divided by 102 and excludes caching benefits according to HAL.. Harness: SAB Self-Debug. Agent: SAB Self-Debug. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-scienceagentbench-sab-self-debug-overall","datasetType":"public","harness":"SAB Self-Debug","agent":"SAB Self-Debug","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"hal-usaco","area":"coding","label":{"de":"USACO","en":"USACO"},"weight":0.004761904761904762,"coverage":1,"score":80,"measurements":[{"observationId":"curated:hal-usaco-usaco-episodic-semantic-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/usaco","benchmarkId":"hal-usaco","benchmarkVersion":"HAL 307-problem set","comparisonCohort":"hal-usaco-usaco-episodic-semantic-overall","retrievedAt":"2026-08-06","value":64.82,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":80,"participants":6,"methodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 0.25172638436482087. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-06-bb1515526aff. Prüfsumme der Quelldaten: bb1515526afffb3e23aa4d2f72a69581fe5da87c2304816379f3f0c0afd3359b. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: high. Cost per task in USD: 0.25172638436482087. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-06-bb1515526aff. Source payload hash: bb1515526afffb3e23aa4d2f72a69581fe5da87c2304816379f3f0c0afd3359b. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits."},"scoreRunConfiguration":{"de":"USACO Episodic + Semantic auf demselben 307-Aufgaben-Split. Kosten sind durch 307 geteilt und berücksichtigen keinen Cache.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning-Stufe: high","en":"USACO Episodic + Semantic on the same 307-problem split. Total cost is divided by 307 and excludes caching benefits.. Harness: USACO Episodic + Semantic. Agent: USACO Episodic + Semantic. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-usaco-usaco-episodic-semantic-overall","datasetType":"public","harness":"USACO Episodic + Semantic","agent":"USACO Episodic + Semantic","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":30.072188449848024,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:84b49308-6b93-47aa-a4f6-776ee1a1e8cd","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=0:suffix=High","retrievedAt":"2026-10-19","value":69.2485549132948,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.357142857142858,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/84b49308-6b93-47aa-a4f6-776ee1a1e8cd/o4-mini/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini (High). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High. Angezeigter Wert: 69,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/84b49308-6b93-47aa-a4f6-776ee1a1e8cd/o4-mini/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini (High). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High. Score display: 69.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: o4-mini (High) (Release o4-mini, Zeile o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: High","en":"Exact AA configuration: o4-mini (High) (release o4-mini, row o4-mini, UUID 84b49308-6b93-47aa-a4f6-776ee1a1e8cd, reasoning=true, source reasoning-token configuration budget=0, suffix=High). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: High"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"High","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":79.665,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.78723404255319,"participants":48,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 19.718. Standardfehler: 0.967. Kosten je Aufgabe in USD: 0.010516. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 19.718. Standard error: 0.967. Cost per task in USD: 0.010516. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"prbench-finance","area":"finance","label":{"de":"PRBench Finance","en":"PRBench Finance"},"weight":0.01020408163265306,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"curated:scale-prbench-finance-o4-mini","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-finance","benchmarkId":"prbench-finance","benchmarkVersion":"full finance set","comparisonCohort":"scale-prbench-finance-full","retrievedAt":"2026-08-26","value":39.22,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":26.666666666666668,"participants":16,"methodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 39,22 % ± 0,51. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-cb85511c79df. Prüfsumme der Quelldaten: cb85511c79df577c828d76d1634d384655e39073f2581851974791a5d6b35d8a. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 39.22% ± 0.51. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-cb85511c79df. Source payload hash: cb85511c79df577c828d76d1634d384655e39073f2581851974791a5d6b35d8a. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"600 finance tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"600 Finanzaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"600 finance tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-finance-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"prbench-legal","area":"legal","label":{"de":"PRBench Legal","en":"PRBench Legal"},"weight":0.017857142857142856,"coverage":1,"score":13.333333333333334,"measurements":[{"observationId":"curated:scale-prbench-legal-o4-mini","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/prbench-legal","benchmarkId":"prbench-legal","benchmarkVersion":"full legal set","comparisonCohort":"scale-prbench-legal-full","retrievedAt":"2026-08-26","value":38.11,"unit":"%","metric":"weighted rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":13.333333333333334,"participants":16,"methodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 38,11 % ± 0,69. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-b3b5c9b3fda5. Prüfsumme der Quelldaten: b3b5c9b3fda5d72967ecd21d2e35d5f69dbc954956c56834f6c4ae03b83b0bc5. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 38.11% ± 0.69. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-b3b5c9b3fda5. Source payload hash: b3b5c9b3fda5d72967ecd21d2e35d5f69dbc954956c56834f6c4ae03b83b0bc5. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.","en":"500 legal tasks with task-specific rubrics and o4-mini as judge."},"scoreRunConfiguration":{"de":"500 Rechtsaufgaben mit aufgabenspezifischen Rubriken und o4-mini als Juror.. Versuche: pass@1","en":"500 legal tasks with task-specific rubrics and o4-mini as judge.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-prbench-legal-full","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"scicode","area":"coding","label":{"de":"SciCode","en":"SciCode"},"weight":0.004761904761904762,"coverage":1,"score":50,"measurements":[{"observationId":"curated:hal-scicode-scicode-tool-calling-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/scicode","benchmarkId":"hal-scicode","benchmarkVersion":"65-problem main set","comparisonCohort":"hal-scicode-scicode-tool-calling-overall","retrievedAt":"2026-08-05","value":4.62,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":50,"participants":5,"methodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 1.0184615384615385. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-b472cbfe2c95. Prüfsumme der Quelldaten: b472cbfe2c95a66fd0271433d74911c832bdcc54f61d0da090226790012d7a6c. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: high. Cost per task in USD: 1.0184615384615385. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-b472cbfe2c95. Source payload hash: b472cbfe2c95a66fd0271433d74911c832bdcc54f61d0da090226790012d7a6c. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65."},"scoreRunConfiguration":{"de":"SciCode Tool Calling auf 65 wissenschaftlichen Hauptproblemen. Kosten sind durch 65 geteilt.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning-Stufe: high","en":"SciCode Tool Calling on 65 main scientific problems. Total cost is divided by 65.. Harness: SciCode Tool Calling. Agent: SciCode Tool Calling. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-scicode-scicode-tool-calling-overall","datasetType":"public","harness":"SciCode Tool Calling","agent":"SciCode Tool Calling","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":100,"measurements":[{"observationId":"curated:hal-swebench-verified-mini-swe-agent-overall-o4-mini-high","sourceId":"princeton-hal","sourceName":"Princeton HAL","sourceUrl":"https://hal.cs.princeton.edu/swebench_verified_mini","benchmarkId":"hal-swebench-verified-mini","benchmarkVersion":"50-task random verified subset","comparisonCohort":"hal-swebench-verified-mini-swe-agent-overall","retrievedAt":"2026-08-05","value":50,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Princeton HAL","higherIsBetter":true,"percentile":100,"participants":5,"methodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Princeton HAL. Datensatztyp: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: high. Kosten je Aufgabe in USD: 4.9692. Quellenkennung: princeton-hal. Kennung der Evidenzaufnahme: curated-princeton-hal-2026-08-05-c086e691d646. Prüfsumme der Quelldaten: c086e691d646f8caddf8f5709e5b6a47fe21605fe94bfa4b76b9262d3603a2a0. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Princeton HAL. Dataset type: public. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: high. Cost per task in USD: 4.9692. Source ID: princeton-hal. Evidence capture ID: curated-princeton-hal-2026-08-05-c086e691d646. Source payload hash: c086e691d646f8caddf8f5709e5b6a47fe21605fe94bfa4b76b9262d3603a2a0. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50."},"scoreRunConfiguration":{"de":"SWE-Agent auf demselben zufälligen 50-Aufgaben-Ausschnitt. Kosten sind durch 50 geteilt.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning-Stufe: high","en":"SWE-Agent on the same random 50-task subset. Total cost is divided by 50.. Harness: SWE-Agent. Agent: SWE-Agent. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"hal-swebench-verified-mini-swe-agent-overall","datasetType":"public","harness":"SWE-Agent","agent":"SWE-Agent","toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":28.37837837837838,"measurements":[{"observationId":"vals-corp-fin-v2:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":58.974,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.37837837837838,"participants":75,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Reasoning-Stufe: high. Maximale Ausgabetoken: 16384. Latenz in Sekunden: 17.528. Standardfehler: 0.967. Kosten je Aufgabe in USD: 0.074104. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Reasoning effort: high. Maximum output tokens: 16384. Latency in seconds: 17.528. Standard error: 0.967. Cost per task in USD: 0.074104. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 16384","en":"Reasoning effort: high. Maximum output tokens: 16384"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":50.64935064935065,"measurements":[{"observationId":"vals-lcb:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":82.208,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.64935064935065,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 32.84. Standardfehler: 1.044. Kosten je Aufgabe in USD: 0.013647. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 32.84. Standard error: 1.044. Cost per task in USD: 0.013647. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":26.923076923076923,"measurements":[{"observationId":"vals-legal-bench:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":79.185,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.923076923076923,"participants":79,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 4.03. Standardfehler: 0.498. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 4.03. Standard error: 0.498. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-math500","area":"math","label":{"de":"MATH 500","en":"MATH 500"},"weight":0.014285714285714285,"coverage":1,"score":77.27272727272727,"measurements":[{"observationId":"vals-math500:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","benchmarkId":"vals-math500","benchmarkVersion":"1","comparisonCohort":"vals-math500:1:overall","retrievedAt":"2026-10-15","value":94.2,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.27272727272727,"participants":23,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 12.536. Standardfehler: 1.045. Kosten je Aufgabe in USD: 0.004784. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-math500-2026-10-15-435ec4da1112. Prüfsumme der Quelldaten: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 12.536. Standard error: 1.045. Cost per task in USD: 0.004784. Source ID: vals-ai. Evidence capture ID: vals-math500-2026-10-15-435ec4da1112. Source payload hash: 435ec4da1112362c0a6bb4afeab53bf7b0faf2000e1ddbe1a878735dca1fe35b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-math500:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":16.417910447761194,"measurements":[{"observationId":"vals-medcode:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":33.791,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.417910447761194,"participants":68,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 21.062. Standardfehler: 2.021. Kosten je Aufgabe in USD: 0.017605. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 21.062. Standard error: 2.021. Cost per task in USD: 0.017605. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medqa","area":"health","label":{"de":"MedQA","en":"MedQA"},"weight":0.023809523809523808,"coverage":1,"score":84.61538461538461,"measurements":[{"observationId":"vals-medqa:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","benchmarkId":"vals-medqa","benchmarkVersion":"1","comparisonCohort":"vals-medqa:1:overall","retrievedAt":"2026-10-15","value":96.017,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.61538461538461,"participants":40,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-04-16. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 6.604. Standardfehler: 0.178. Kosten je Aufgabe in USD: 0.002784. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medqa-2026-10-15-d67905978d93. Prüfsumme der Quelldaten: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-04-16. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 6.604. Standard error: 0.178. Cost per task in USD: 0.002784. Source ID: vals-ai. Evidence capture ID: vals-medqa-2026-10-15-d67905978d93. Source payload hash: d67905978d932f64698ebe9e230d9519e5bae1c9d7176dda8379dc3874b6f90f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-medqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":10.294117647058824,"measurements":[{"observationId":"vals-medscribe:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":69.139,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.294117647058824,"participants":69,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 81.959. Standardfehler: 1.957. Kosten je Aufgabe in USD: 0.040605. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 81.959. Standard error: 1.957. Cost per task in USD: 0.040605. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mgsm","area":"math","label":{"de":"MGSM","en":"MGSM"},"weight":0.014285714285714285,"coverage":1,"score":96.55172413793103,"measurements":[{"observationId":"vals-mgsm:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","benchmarkId":"vals-mgsm","benchmarkVersion":"1","comparisonCohort":"vals-mgsm:1:overall","retrievedAt":"2026-10-15","value":93.418,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":96.55172413793103,"participants":30,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-01-09. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 8.067. Standardfehler: 1.507. Kosten je Aufgabe in USD: 0.002905. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mgsm-2026-10-15-1b87f953e52c. Prüfsumme der Quelldaten: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-01-09. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 8.067. Standard error: 1.507. Cost per task in USD: 0.002905. Source ID: vals-ai. Evidence capture ID: vals-mgsm-2026-10-15-1b87f953e52c. Source payload hash: 1b87f953e52c00bf4bdd90c3add26165f644eeb7a74a8382d2a0324ff0cc1040. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mgsm:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":27.272727272727273,"measurements":[{"observationId":"vals-mmlu-pro:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":80.561,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.272727272727273,"participants":78,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 10.443. Standardfehler: 0.386. Kosten je Aufgabe in USD: 0.005524. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 10.443. Standard error: 0.386. Cost per task in USD: 0.005524. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":34.61538461538461,"measurements":[{"observationId":"vals-mortgage-tax:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":64.826,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.61538461538461,"participants":53,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 12.996. Standardfehler: 0.944. Kosten je Aufgabe in USD: 0.011654. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 12.996. Standard error: 0.944. Cost per task in USD: 0.011654. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":27.586206896551722,"measurements":[{"observationId":"vals-sage:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":41.061,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.586206896551722,"participants":59,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: high. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 27.215. Standardfehler: 3.395. Kosten je Aufgabe in USD: 0.02941. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: high. Maximum output tokens: 30000. Latency in seconds: 27.215. Standard error: 3.395. Cost per task in USD: 0.02941. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 30000","en":"Reasoning effort: high. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":75,"measurements":[{"observationId":"vals-tax-eval-v2:overall:openai/o4-mini-2025-04-16","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":74.776,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75,"participants":77,"methodology":{"de":"Kennung des Quellmodells: openai/o4-mini-2025-04-16. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Modellstand: o4-mini-2025-04-16. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: high. Maximale Ausgabetoken: 100000. Latenz in Sekunden: 15.447. Standardfehler: 0.853. Kosten je Aufgabe in USD: 0.007057. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: openai/o4-mini-2025-04-16. Source model name: o4-mini. Provider reported by source: OpenAI. Model snapshot: o4-mini-2025-04-16. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: high. Maximum output tokens: 100000. Latency in seconds: 15.447. Standard error: 0.853. Cost per task in USD: 0.007057. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: high. Maximale Ausgabetoken: 100000","en":"Reasoning effort: high. Maximum output tokens: 100000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"vista","area":"multimodal","label":{"de":"VISTA","en":"VISTA"},"weight":0.017857142857142856,"coverage":1,"score":76.92307692307692,"measurements":[{"observationId":"curated:scale-vista-o4-mini-high","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/visual_language_understanding","benchmarkId":"vista","benchmarkVersion":"758 private prompt-image pairs","comparisonCohort":"scale-vista-private-overall","retrievedAt":"2026-08-26","value":51.79,"unit":"%","metric":"rubric score","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":76.92307692307692,"participants":14,"methodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Reasoning-Stufe: high. Angezeigter Wert: 51,79 % ± 0,63. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-d9e1645c285f. Prüfsumme der Quelldaten: d9e1645c285ffd7325c4e563c683e9f1696b6e313ce4a86f2aec17bea03aea43. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Reasoning effort: high. Score display: 51.79% ± 0.63. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-d9e1645c285f. Source payload hash: d9e1645c285ffd7325c4e563c683e9f1696b6e313ce4a86f2aec17bea03aea43. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.","en":"758 single-turn image-text tasks with structured yes-no rubrics."},"scoreRunConfiguration":{"de":"758 einzelne Bild-Text-Aufgaben mit strukturierten Ja-Nein-Rubriken.. Versuche: pass@1. Reasoning-Stufe: high","en":"758 single-turn image-text tasks with structured yes-no rubrics.. Attempts: pass@1. Reasoning effort: high"},"scoreComparisonSetup":{"publicationCohort":"scale-vista-private-overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":"high","weight":1,"configurationsDiffer":true}]},{"familyId":"visual-toolbench","area":"multimodal","label":{"de":"VisualToolBench","en":"VisualToolBench"},"weight":0.017857142857142856,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"curated:scale-visual-toolbench-o4-mini","sourceId":"scale-ai","sourceName":"Scale AI","sourceUrl":"https://labs.scale.com/leaderboard/vtb","benchmarkId":"visual-toolbench","benchmarkVersion":"1,204-task release","comparisonCohort":"scale-visual-toolbench-apr","retrievedAt":"2026-08-26","value":11.12,"unit":"%","metric":"average pass rate","task":"overall","evaluator":"Scale AI","higherIsBetter":true,"percentile":44.44444444444444,"participants":10,"methodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Kennung des Quellmodells: curated/o4-mini. Name des Quellmodells: o4-mini. Anbieter laut Quelle: OpenAI. Evaluator: Scale AI. Datensatztyp: public. Versuche: pass@1. Angezeigter Wert: 11,12 % ± 0,08. Quellenkennung: scale-ai. Kennung der Evidenzaufnahme: curated-scale-ai-2026-08-26-257d5236c47b. Prüfsumme der Quelldaten: 257d5236c47bbd830a034d02fa519f4460680db0bc46903d64a6d61d4dfa924d. Parserversion: curated-v1. Importiert am: 2026-10-15","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Source model ID: curated/o4-mini. Source model name: o4-mini. Provider reported by source: OpenAI. Evaluator: Scale AI. Dataset type: public. Attempts: pass@1. Score display: 11.12% ± 0.08. Source ID: scale-ai. Evidence capture ID: curated-scale-ai-2026-08-26-257d5236c47b. Source payload hash: 257d5236c47bbd830a034d02fa519f4460680db0bc46903d64a6d61d4dfa924d. Parser version: curated-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.","en":"1,204 tasks, six tools, and at most 20 tool calls per task."},"scoreRunConfiguration":{"de":"1.204 Aufgaben, sechs Werkzeuge und höchstens 20 Werkzeugaufrufe pro Aufgabe.. Versuche: pass@1","en":"1,204 tasks, six tools, and at most 20 tool calls per task.. Attempts: pass@1"},"scoreComparisonSetup":{"publicationCohort":"scale-visual-toolbench-apr","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":"pass@1"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":25,"totalFamilies":71,"sourceCount":4,"coverage":0.3423469387755102,"lowerBound":14.567912064035117,"upperBound":80.33321818648409,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-08-05","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.3000000000000001,"lowerBound":19.37662337662338,"upperBound":89.37662337662339,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.07142857142857142,"lowerBound":1.7387218045112782,"upperBound":94.59586466165413,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":2,"totalFamilies":5,"sourceCount":1,"coverage":0.4,"lowerBound":34.76489028213166,"upperBound":94.76489028213166,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":9.090909090909092,"upperBound":75.75757575757576,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":4.166666666666667,"upperBound":87.50000000000001,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":12.463768115942031,"upperBound":92.46376811594203,"provisional":true,"configurationsDiffer":true},"finance":{"score":41.165107415107414,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.5714285714285714,"lowerBound":23.52291852291852,"upperBound":66.38006138006138,"provisional":true,"configurationsDiffer":true},"legal":{"score":20.128205128205128,"coveredFamilies":2,"totalFamilies":4,"sourceCount":2,"coverage":0.5,"lowerBound":10.064102564102564,"upperBound":60.06410256410256,"provisional":true,"configurationsDiffer":true},"health":{"score":37.109137570068214,"coveredFamilies":3,"totalFamilies":3,"sourceCount":1,"coverage":1,"lowerBound":37.109137570068214,"upperBound":37.109137570068214,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":50.47990327245646,"coveredFamilies":3,"totalFamilies":4,"sourceCount":3,"coverage":0.75,"lowerBound":37.85992745434235,"upperBound":62.85992745434235,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":13.793103448275861,"upperBound":63.79310344827586,"provisional":true,"configurationsDiffer":true}}},{"modelId":"openbmb-minicpm5-1b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"openbmb-minicpm5-2b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"perplexity-sonar","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"perplexity-sonar-deep-research","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"perplexity-sonar-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"perplexity-sonar-reasoning-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"phi-4","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":0,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:ee708f92-374e-4123-b900-e22d7b2afc19","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":23.5374149659864,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":0,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Phi-4 (Release phi-4, Zeile phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/ee708f92-374e-4123-b900-e22d7b2afc19/phi-4/phi-4. Name des Quellmodells: Phi-4. Anbieter laut Quelle: Microsoft. Modellstand: Phi-4. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 23,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Phi-4 (release phi-4, row phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/ee708f92-374e-4123-b900-e22d7b2afc19/phi-4/phi-4. Source model name: Phi-4. Provider reported by source: Microsoft. Model snapshot: Phi-4. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 23.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Phi-4 (Release phi-4, Zeile phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Phi-4 (release phi-4, row phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Phi-4 (Release phi-4, Zeile phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Phi-4 (release phi-4, row phi-4, UUID ee708f92-374e-4123-b900-e22d7b2afc19, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0,"upperBound":98.57142857142858,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":0,"upperBound":80,"provisional":true,"configurationsDiffer":true}}},{"modelId":"poolside-laguna-m.1","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"poolside-laguna-xs.2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-2.5-72b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-2.5-7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-2.5-coder-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-2.5-vl-72b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-14b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-235b-a22b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-235b-a22b-base","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-235b-a22b-thinking","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-30b-a3b-base","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-30b-a3b-thinking","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-coder","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-coder-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-coder-next","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":24.390243902439025,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":403.62,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":24.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":15.853658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":47,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.853658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 47,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 47.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":1.0593550691766298,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 1,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 1.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":2.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 2,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 2.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":513.94,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":17.391304347826086,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":35.2380952380952,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.391304347826086,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 35,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 35.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":4.878048780487805,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":-62.4,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":4.878048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":13.414634146341463,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":10.1482854494903,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":13.414634146341463,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 10,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 10.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:fc92f822-04b7-420d-9c07-a21af5e9aac7","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Name des Quellmodells: Qwen 3 Coder Next. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Coder Next. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: non-reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/fc92f822-04b7-420d-9c07-a21af5e9aac7/qwen3-coder-next/qwen3-coder-next. Source model name: Qwen 3 Coder Next. Provider reported by source: Alibaba. Model snapshot: Qwen3 Coder Next. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: non-reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Coder Next (Release qwen3-coder-next, Zeile qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Coder Next (release qwen3-coder-next, row qwen3-coder-next, UUID fc92f822-04b7-420d-9c07-a21af5e9aac7, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":1,"coverage":0.083843537414966,"lowerBound":1.215553559705961,"upperBound":92.83119981820937,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":5.836236933797909,"upperBound":70.1219512195122,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":1.6260162601626016,"upperBound":68.29268292682927,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":5.487804878048781,"upperBound":80.48780487804878,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":3.4782608695652173,"upperBound":83.47826086956522,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3-next-80b-a3b-instruct","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":28.985507246376812,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:2698f6c6-e436-47ce-a583-dbc25596c571","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:non-reasoning:tokens=0:suffix=unspecified","retrievedAt":"2026-10-19","value":39.6598639455782,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.985507246376812,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 Next 80B A3B Instruct (Release qwen3-next-80b-a3b-instruct, Zeile qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/2698f6c6-e436-47ce-a583-dbc25596c571/qwen3-next-80b-a3b-instruct/qwen3-next-80b-a3b-instruct. Name des Quellmodells: Qwen 3 Next 80B A3B Instruct. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 Next 80B A3B Instruct. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning. Angezeigter Wert: 39,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 Next 80B A3B Instruct (release qwen3-next-80b-a3b-instruct, row qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/2698f6c6-e436-47ce-a583-dbc25596c571/qwen3-next-80b-a3b-instruct/qwen3-next-80b-a3b-instruct. Source model name: Qwen 3 Next 80B A3B Instruct. Provider reported by source: Alibaba. Model snapshot: Qwen3 Next 80B A3B Instruct. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning. Score display: 39.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 Next 80B A3B Instruct (Release qwen3-next-80b-a3b-instruct, Zeile qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Next 80B A3B Instruct (release qwen3-next-80b-a3b-instruct, row qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 Next 80B A3B Instruct (Release qwen3-next-80b-a3b-instruct, Zeile qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: non-reasoning","en":"Exact AA configuration: Qwen3 Next 80B A3B Instruct (release qwen3-next-80b-a3b-instruct, row qwen3-next-80b-a3b-instruct, UUID 2698f6c6-e436-47ce-a583-dbc25596c571, reasoning=false, source reasoning-token configuration budget=0, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: non-reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.41407867494824024,"upperBound":98.98550724637683,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":5.797101449275363,"upperBound":85.79710144927536,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3-next-80b-a3b-thinking","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-vl-235b-a22b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-vl-235b-a22b-thinking","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":44.927536231884055,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:a803d3d0-d22e-49a0-ac2c-b9c6f1141065","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":56.4625850340136,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":44.927536231884055,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/a803d3d0-d22e-49a0-ac2c-b9c6f1141065/qwen3-vl-235b-a22b/qwen3-vl-235b-a22b-reasoning. Name des Quellmodells: Qwen 3 VL 235B A22B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 235B A22B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 56,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/a803d3d0-d22e-49a0-ac2c-b9c6f1141065/qwen3-vl-235b-a22b/qwen3-vl-235b-a22b-reasoning. Source model name: Qwen 3 VL 235B A22B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 235B A22B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 56.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":28.571428571428573,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:a803d3d0-d22e-49a0-ac2c-b9c6f1141065","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":68.728323699422,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.571428571428573,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/a803d3d0-d22e-49a0-ac2c-b9c6f1141065/qwen3-vl-235b-a22b/qwen3-vl-235b-a22b-reasoning. Name des Quellmodells: Qwen 3 VL 235B A22B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 235B A22B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 68,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/a803d3d0-d22e-49a0-ac2c-b9c6f1141065/qwen3-vl-235b-a22b/qwen3-vl-235b-a22b-reasoning. Source model name: Qwen 3 VL 235B A22B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 235B A22B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 68.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 235B A22B (Reasoning) (Release qwen3-vl-235b-a22b, Zeile qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 235B A22B (Reasoning) (release qwen3-vl-235b-a22b, row qwen3-vl-235b-a22b-reasoning, UUID a803d3d0-d22e-49a0-ac2c-b9c6f1141065, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":2,"totalFamilies":71,"sourceCount":1,"coverage":0.02321428571428572,"lowerBound":0.8969239869860989,"upperBound":98.57549541555753,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":8.98550724637681,"upperBound":88.98550724637681,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":3.5714285714285716,"upperBound":91.07142857142857,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3-vl-30b-a3b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-vl-30b-a3b-thinking","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":40.57971014492754,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:ce3d286e-093d-413d-a81a-0270309f039e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":45.1020408163265,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.57971014492754,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/ce3d286e-093d-413d-a81a-0270309f039e/qwen3-vl-30b-a3b/qwen3-vl-30b-a3b-reasoning. Name des Quellmodells: Qwen 3 VL 30B A3B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 30B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 45,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/ce3d286e-093d-413d-a81a-0270309f039e/qwen3-vl-30b-a3b/qwen3-vl-30b-a3b-reasoning. Source model name: Qwen 3 VL 30B A3B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 30B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 45.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":21.428571428571427,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:ce3d286e-093d-413d-a81a-0270309f039e","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":61.849710982659,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.428571428571427,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/ce3d286e-093d-413d-a81a-0270309f039e/qwen3-vl-30b-a3b/qwen3-vl-30b-a3b-reasoning. Name des Quellmodells: Qwen 3 VL 30B A3B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 30B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 61,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/ce3d286e-093d-413d-a81a-0270309f039e/qwen3-vl-30b-a3b/qwen3-vl-30b-a3b-reasoning. Source model name: Qwen 3 VL 30B A3B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 30B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 61.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 30B A3B (Reasoning) (Release qwen3-vl-30b-a3b, Zeile qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 30B A3B (Reasoning) (release qwen3-vl-30b-a3b, row qwen3-vl-30b-a3b-reasoning, UUID ce3d286e-093d-413d-a81a-0270309f039e, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":2,"totalFamilies":71,"sourceCount":1,"coverage":0.02321428571428572,"lowerBound":0.7710366755397812,"upperBound":98.4496081041112,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":8.115942028985508,"upperBound":88.1159420289855,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":2.6785714285714284,"upperBound":90.17857142857143,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3-vl-32b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-vl-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3-vl-8b-thinking","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":30.434782608695652,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:dec8073c-57e2-41c0-b1aa-7a62960f103f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":39.8639455782313,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.434782608695652,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/dec8073c-57e2-41c0-b1aa-7a62960f103f/qwen3-vl-8b/qwen3-vl-8b-reasoning. Name des Quellmodells: Qwen 3 VL 8B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 8B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 39,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/dec8073c-57e2-41c0-b1aa-7a62960f103f/qwen3-vl-8b/qwen3-vl-8b-reasoning. Source model name: Qwen 3 VL 8B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 8B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 39.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":14.285714285714286,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:dec8073c-57e2-41c0-b1aa-7a62960f103f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":56.647398843930596,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.285714285714286,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/dec8073c-57e2-41c0-b1aa-7a62960f103f/qwen3-vl-8b/qwen3-vl-8b-reasoning. Name des Quellmodells: Qwen 3 VL 8B Thinking. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3 VL 8B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 56,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/dec8073c-57e2-41c0-b1aa-7a62960f103f/qwen3-vl-8b/qwen3-vl-8b-reasoning. Source model name: Qwen 3 VL 8B Thinking. Provider reported by source: Alibaba. Model snapshot: Qwen3 VL 8B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 56.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3 VL 8B (Reasoning) (Release qwen3-vl-8b, Zeile qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3 VL 8B (Reasoning) (release qwen3-vl-8b, row qwen3-vl-8b-reasoning, UUID dec8073c-57e2-41c0-b1aa-7a62960f103f, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":2,"totalFamilies":71,"sourceCount":1,"coverage":0.02321428571428572,"lowerBound":0.5623336291038156,"upperBound":98.24090505767525,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6.086956521739131,"upperBound":86.08695652173913,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":1.7857142857142858,"upperBound":89.28571428571429,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.5-122b-a10b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":497.34,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":76.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 76,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 76.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":35.36585365853659,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":5.481116230292565,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.36585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 5,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 5.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":3.4000000000000004,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 3,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 3.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":34.146341463414636,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":814.01,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.146341463414636,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":19.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":-41.5,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":19.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":35.36585365853659,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":25.2085264133457,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":35.36585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 25,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 25.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:2236df45-0699-40d1-b5cc-69ee345d2257","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Name des Quellmodells: Qwen 3.5 122B A10B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2236df45-0699-40d1-b5cc-69ee345d2257/qwen3-5-122b-a10b/qwen3-5-122b-a10b. Source model name: Qwen 3.5 122B A10B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 122B A10B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 122B A10B (Reasoning) (Release qwen3-5-122b-a10b, Zeile qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 122B A10B (Reasoning) (release qwen3-5-122b-a10b, row qwen3-5-122b-a10b, UUID 2236df45-0699-40d1-b5cc-69ee345d2257, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":1.9273685083789613,"upperBound":94.97158619545378,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":11.759581881533101,"upperBound":76.04529616724739,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":6.504065040650406,"upperBound":73.17073170731707,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":8.130081300813009,"upperBound":83.130081300813,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.5-27b","contributions":[{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":34.285714285714285,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":35.4990583804143,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.285714285714285,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 27B (Reasoning) (Release qwen3-5-27b, Zeile qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67/qwen3-5-27b/qwen3-5-27b. Name des Quellmodells: Qwen 3.5 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 35,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 27B (Reasoning) (release qwen3-5-27b, row qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67/qwen3-5-27b/qwen3-5-27b. Source model name: Qwen 3.5 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 35.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 27B (Reasoning) (Release qwen3-5-27b, Zeile qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 27B (Reasoning) (release qwen3-5-27b, row qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 27B (Reasoning) (Release qwen3-5-27b, Zeile qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 27B (Reasoning) (release qwen3-5-27b, row qwen3-5-27b, UUID 169e47f5-3d4d-4ad4-8f8b-ab46f0c73f67, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.005952380952380953,"lowerBound":0.20408163265306123,"upperBound":99.60884353741497,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":2.8571428571428568,"upperBound":94.52380952380952,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.5-35b-a3b","contributions":[{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":2.857142857142857,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:46d8315e-1630-463f-ab62-84185fa0faab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":21.5160075329567,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":2.857142857142857,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 35B A3B (Reasoning) (Release qwen3-5-35b-a3b, Zeile qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/46d8315e-1630-463f-ab62-84185fa0faab/qwen3-5-35b-a3b/qwen3-5-35b-a3b. Name des Quellmodells: Qwen 3.5 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 21,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 35B A3B (Reasoning) (release qwen3-5-35b-a3b, row qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/46d8315e-1630-463f-ab62-84185fa0faab/qwen3-5-35b-a3b/qwen3-5-35b-a3b. Source model name: Qwen 3.5 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 21.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 35B A3B (Reasoning) (Release qwen3-5-35b-a3b, Zeile qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 35B A3B (Reasoning) (release qwen3-5-35b-a3b, row qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 35B A3B (Reasoning) (Release qwen3-5-35b-a3b, Zeile qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 35B A3B (Reasoning) (release qwen3-5-35b-a3b, row qwen3-5-35b-a3b, UUID 46d8315e-1630-463f-ab62-84185fa0faab, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.005952380952380953,"lowerBound":0.017006802721088437,"upperBound":99.421768707483,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"agentic":{"score":null,"coveredFamilies":1,"totalFamilies":12,"sourceCount":1,"coverage":0.08333333333333333,"lowerBound":0.23809523809523808,"upperBound":91.9047619047619,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.5-397b-a17b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":31.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":562.4,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":77.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 77,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 77.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":34.61538461538461,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":15.339233038348098,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.61538461538461,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 15,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 15.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":6.886670023070867,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 6,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 6.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":55.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":12.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 12,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 12.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":32.926829268292686,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":792.97,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":32.926829268292686,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":28.571428571428573,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":34.0866290018832,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.571428571428573,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 34,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 34.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":25.609756097560975,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":-30.75,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":25.609756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":40.24390243902439,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":28.962001853568097,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.24390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 29,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 29.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:0e66bae9-41f1-42fc-9276-ce8cb6f72919","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=3186:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Name des Quellmodells: Qwen 3.5 397B A17B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/0e66bae9-41f1-42fc-9276-ce8cb6f72919/qwen3-5-397b-a17b/qwen3-5-397b-a17b. Source model name: Qwen 3.5 397B A17B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 397B A17B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 397B A17B (Reasoning) (Release qwen3-5-397b-a17b, Zeile qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 397B A17B (Reasoning) (release qwen3-5-397b-a17b, row qwen3-5-397b-a17b, UUID 0e66bae9-41f1-42fc-9276-ce8cb6f72919, reasoning=true, source reasoning-token configuration budget=3186, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":1,"coverage":0.08146258503401363,"lowerBound":2.9287915916835785,"upperBound":94.78253308828222,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":17.770034843205575,"upperBound":82.05574912891987,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":8.536585365853659,"upperBound":75.20325203252033,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":14.107031180201913,"upperBound":72.44036451353524,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.5-9b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":340.67,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":27.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":70,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":27.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 70,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 70.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":1.0473867343065657,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":34.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":6.6000000000000005,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 6,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 6.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":17.073170731707318,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":419.52,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.073170731707318,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":6.097560975609756,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":-53.46666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":6.097560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":20.73170731707317,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":14.921223354958299,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20.73170731707317,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 14,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 14.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":40.853658536585364,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:2d28a13a-096e-475a-beb8-26bbd1c7d51c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Reasoning","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Name des Quellmodells: Qwen 3.5 9B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/2d28a13a-096e-475a-beb8-26bbd1c7d51c/qwen3-5-9b/qwen3-5-9b. Source model name: Qwen 3.5 9B. Provider reported by source: Alibaba. Model snapshot: Qwen3.5 9B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.5 9B (Reasoning) (Release qwen3-5-9b, Zeile qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.5 9B (Reasoning) (release qwen3-5-9b, row qwen3-5-9b, UUID 2d28a13a-096e-475a-beb8-26bbd1c7d51c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":1.32404181184669,"upperBound":94.3682594989215,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":1.3617886178861793,"upperBound":98.02845528455285,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":10.365853658536585,"upperBound":74.65156794425087,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":2.032520325203252,"upperBound":68.69918699186992,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":4.776422764227643,"upperBound":79.77642276422765,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.6-27b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":815.82,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":48.78048780487805,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":77.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":48.78048780487805,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 77,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 77.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":11.045223588713588,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 11,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 11.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":46.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":11,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 11,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 11.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":42.68292682926829,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":985.63,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.68292682926829,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":31.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":-20.016666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":31.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":23.0769230769231,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":31.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 23,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 23.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":26.271186440677965,"measurements":[{"observationId":"vals-swebench:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":70,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.271186440677965,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1028.733. Standardfehler: 2.051. Kosten je Aufgabe in USD: 0.91414. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1028.733. Standard error: 2.051. Cost per task in USD: 0.91414. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":27.845528455284555,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:8c29d66d-bf98-4ea3-8572-5409353ecc66","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=5676:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8c29d66d-bf98-4ea3-8572-5409353ecc66/qwen3-6-27b/qwen3-6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 27B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 27B (Reasoning) (Release qwen3-6-27b, Zeile qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 27B (Reasoning) (release qwen3-6-27b, row qwen3-6-27b, UUID 8c29d66d-bf98-4ea3-8572-5409353ecc66, reasoning=true, source reasoning-token configuration budget=5676, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":44.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.333333333333336,"participants":43,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2007.631. Standardfehler: 5.303. Kosten je Aufgabe in USD: 0.298145. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.6. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 2007.631. Standard error: 5.303. Cost per task in USD: 0.298145. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.6. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 0.6. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":56.75675675675676,"measurements":[{"observationId":"vals-corp-fin-v2:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":62.315,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.75675675675676,"participants":75,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 62.759. Standardfehler: 0.954. Kosten je Aufgabe in USD: 0.053549. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 62.759. Standard error: 0.954. Cost per task in USD: 0.053549. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":73.07692307692308,"measurements":[{"observationId":"vals-mortgage-tax:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":68.283,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.07692307692308,"participants":53,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 42.074. Standardfehler: 0.916. Kosten je Aufgabe in USD: 0.016906. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 42.074. Standard error: 0.916. Cost per task in USD: 0.016906. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":44.827586206896555,"measurements":[{"observationId":"vals-sage:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":45.577,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.827586206896555,"participants":59,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 159.25. Standardfehler: 3.427. Kosten je Aufgabe in USD: 0.050269. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 159.25. Standard error: 3.427. Cost per task in USD: 0.050269. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":34.21052631578947,"measurements":[{"observationId":"vals-tax-eval-v2:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":71.26,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.21052631578947,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 147.778. Standardfehler: 0.882. Kosten je Aufgabe in USD: 0.030256. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 147.778. Standard error: 0.882. Cost per task in USD: 0.030256. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":15.277777777777779,"measurements":[{"observationId":"vals-vibe-code-bench:overall:alibaba/qwen3.6-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":11.941,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.277777777777779,"participants":73,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.6-27b. Name des Quellmodells: Qwen 3.6 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 9762.856. Standardfehler: 3.414. Kosten je Aufgabe in USD: 4.655127. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.6-27b. Source model name: Qwen 3.6 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 9762.856. Standard error: 3.414. Cost per task in USD: 4.655127. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":14,"totalFamilies":71,"sourceCount":2,"coverage":0.14382086167800456,"lowerBound":6.168881389522246,"upperBound":91.78679522172179,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":2,"coverage":0.14444444444444451,"lowerBound":3.1318037756648764,"upperBound":88.68735933122043,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":15.940766550522648,"upperBound":80.22648083623695,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":10.56910569105691,"upperBound":77.23577235772358,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":10.873983739837398,"upperBound":85.8739837398374,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.4285714285714286,"lowerBound":23.43488659278133,"upperBound":80.57774373563846,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":22.413793103448278,"upperBound":72.41379310344828,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.6-35b-a3b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":712.84,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":71.6666666666667,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 71,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 71.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":34.146341463414636,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":5.2085656145058845,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.146341463414636,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 5,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 5.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":34.75609756097561,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":6.6000000000000005,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.75609756097561,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 6,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 6.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":36.58536585365854,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":892.41,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":36.58536585365854,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":-22.183333333333334,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":29.26829268292683,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":22.242817423540302,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":29.26829268292683,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 22,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 22.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:8665ca00-c687-44c7-875c-22618cb31c4f","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=5395:suffix=Reasoning","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Name des Quellmodells: Qwen 3.6 35B A3B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8665ca00-c687-44c7-875c-22618cb31c4f/qwen3-6-35b-a3b/qwen3-6-35b-a3b. Source model name: Qwen 3.6 35B A3B. Provider reported by source: Alibaba. Model snapshot: Qwen3.6 35B A3B (Reasoning). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Reasoning. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.6 35B A3B (Reasoning) (Release qwen3-6-35b-a3b, Zeile qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Reasoning","en":"Exact AA configuration: Qwen3.6 35B A3B (Reasoning) (release qwen3-6-35b-a3b, row qwen3-6-35b-a3b, UUID 8665ca00-c687-44c7-875c-22618cb31c4f, reasoning=true, source reasoning-token configuration budget=5395, suffix=Reasoning). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Reasoning"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Reasoning","weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":2.209432553509209,"upperBound":95.25365024058402,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":11.236933797909407,"upperBound":75.5226480836237,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":10.16260162601626,"upperBound":76.82926829268293,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":8.94308943089431,"upperBound":83.9430894308943,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.7-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3.7-max","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":18.42105263157895,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":18.75,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.42105263157895,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 18,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 18.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":54.8780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":916.31,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":54.8780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":55.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":79,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 79,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 79.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":22.976890336173604,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 23,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 23.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":41.46341463414634,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":8.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":41.46341463414634,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 8,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 8.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":56.09756097560975,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":1127.65,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":56.09756097560975,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":97.10144927536231,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":80.5442176870748,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":97.10144927536231,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 80,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 80.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":62.857142857142854,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":42.4670433145009,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.857142857142854,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 42,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 42.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":71.95121951219512,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":13.483333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":71.95121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":73.02631578947368,"measurements":[{"observationId":"vals-gpqa:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":90.152,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.02631578947368,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 126.611. Standardfehler: 1.497. Kosten je Aufgabe in USD: 0.06149. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 126.611. Standard error: 1.497. Cost per task in USD: 0.06149. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":40.5004633920297,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 40,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 40.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":16.949152542372882,"measurements":[{"observationId":"vals-swebench:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":68.8,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":16.949152542372882,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 976.131. Standardfehler: 2.074. Kosten je Aufgabe in USD: 3.227207. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 976.131. Standard error: 2.074. Cost per task in USD: 3.227207. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":57.79036004645761,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:69534bed-2ffd-4235-832b-e20a810333ab","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2409:suffix=unspecified","retrievedAt":"2026-10-19","value":1.51515151515152,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Max. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/69534bed-2ffd-4235-832b-e20a810333ab/qwen3-7-max/qwen3-7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Max. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Max (Release qwen3-7-max, Zeile qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Max (release qwen3-7-max, row qwen3-7-max, UUID 69534bed-2ffd-4235-832b-e20a810333ab, reasoning=true, source reasoning-token configuration budget=2409, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":59.176,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":43,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 664.843. Standardfehler: 1.35. Kosten je Aufgabe in USD: 0.719164. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 664.843. Standard error: 1.35. Cost per task in USD: 0.719164. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":61.049,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.642857142857146,"participants":57,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 935.196. Standardfehler: 0.375. Kosten je Aufgabe in USD: 0.33233. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 935.196. Standard error: 0.375. Cost per task in USD: 0.33233. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":24.193548387096776,"measurements":[{"observationId":"vals-code-migration:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":13.069,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.193548387096776,"participants":63,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1531.852. Standardfehler: 2.858. Kosten je Aufgabe in USD: 1.932899. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 1531.852. Standard error: 2.858. Cost per task in USD: 1.932899. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":63.513513513513516,"measurements":[{"observationId":"vals-corp-fin-v2:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":63.714,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":63.513513513513516,"participants":75,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 19.934. Standardfehler: 0.946. Kosten je Aufgabe in USD: 0.219883. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 19.934. Standard error: 0.946. Cost per task in USD: 0.219883. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":44.067796610169495,"measurements":[{"observationId":"vals-emb:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":57.041,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.067796610169495,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1957.165. Standardfehler: 2.853. Kosten je Aufgabe in USD: 2.857267. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 1957.165. Standard error: 2.853. Cost per task in USD: 2.857267. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":38.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":47.776,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":38.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 635.178. Standardfehler: 0.727. Kosten je Aufgabe in USD: 0.553214. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 635.178. Standard error: 0.727. Cost per task in USD: 0.553214. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":80.51948051948052,"measurements":[{"observationId":"vals-lcb:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.057,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":80.51948051948052,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 192.175. Standardfehler: 0.974. Kosten je Aufgabe in USD: 0.086751. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 192.175. Standard error: 0.974. Cost per task in USD: 0.086751. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":37.704918032786885,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":1.667,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":37.704918032786885,"participants":62,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1923.25. Standardfehler: 0.833. Kosten je Aufgabe in USD: 1.57592. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 1923.25. Standard error: 0.833. Cost per task in USD: 1.57592. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":75.64102564102564,"measurements":[{"observationId":"vals-legal-bench:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":84.913,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":75.64102564102564,"participants":79,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 11.853. Standardfehler: 0.413. Kosten je Aufgabe in USD: 0.005967. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 11.853. Standard error: 0.413. Cost per task in USD: 0.005967. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":35.833333333333336,"measurements":[{"observationId":"vals-legal-research:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":25.481,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":35.833333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 808.802. Standardfehler: 3.029. Kosten je Aufgabe in USD: 1.400701. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 808.802. Standard error: 3.029. Cost per task in USD: 1.400701. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":25.37313432835821,"measurements":[{"observationId":"vals-medcode:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":38.751,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25.37313432835821,"participants":68,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 28.755. Standardfehler: 2.196. Kosten je Aufgabe in USD: 0.042362. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 28.755. Standard error: 2.196. Cost per task in USD: 0.042362. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":36.76470588235294,"measurements":[{"observationId":"vals-medscribe:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":79.396,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.76470588235294,"participants":69,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 108.046. Standardfehler: 1.907. Kosten je Aufgabe in USD: 0.06907. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 108.046. Standard error: 1.907. Cost per task in USD: 0.06907. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":87.01298701298701,"measurements":[{"observationId":"vals-mmlu-pro:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":89.311,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":87.01298701298701,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 29.15. Standardfehler: 0.305. Kosten je Aufgabe in USD: 0.01678. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 29.15. Standard error: 0.305. Cost per task in USD: 0.01678. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":34.54545454545455,"measurements":[{"observationId":"vals-tax-agent-bench:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":15.832,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":34.54545454545455,"participants":56,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 209.726. Standardfehler: 2.617. Kosten je Aufgabe in USD: 0.435494. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 209.726. Standard error: 2.617. Cost per task in USD: 0.435494. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":84.86842105263158,"measurements":[{"observationId":"vals-tax-eval-v2:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.306,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":84.86842105263158,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 57.677. Standardfehler: 0.839. Kosten je Aufgabe in USD: 0.028122. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 57.677. Standard error: 0.839. Cost per task in USD: 0.028122. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":44.44444444444444,"measurements":[{"observationId":"vals-vibe-code-bench:overall:alibaba/qwen3.7-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":47.671,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":44.44444444444444,"participants":73,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-max. Name des Quellmodells: Qwen 3.7 Max. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 4689.627. Standardfehler: 4.63. Kosten je Aufgabe in USD: 11.400322. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-max. Source model name: Qwen 3.7 Max. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 4689.627. Standard error: 4.63. Cost per task in USD: 11.400322. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":27,"totalFamilies":71,"sourceCount":2,"coverage":0.29512471655328804,"lowerBound":16.164436564561413,"upperBound":86.65196490923262,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":5,"totalFamilies":15,"sourceCount":2,"coverage":0.288888888888889,"lowerBound":13.71937886628375,"upperBound":84.83048997739485,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":23.770172382174948,"upperBound":80.91302952503209,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":79.48210326259107,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":52.988068841727376,"upperBound":86.32140217506071,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":1,"coverage":0.4166666666666667,"lowerBound":20.492695152515434,"upperBound":78.82602848584877,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":19.420289855072465,"upperBound":99.42028985507247,"provisional":true,"configurationsDiffer":true},"finance":{"score":53.06570381102049,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":37.904074150728924,"upperBound":66.4755027221575,"provisional":true,"configurationsDiffer":true},"legal":{"score":49.72642566904862,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":37.294819251786464,"upperBound":62.294819251786464,"provisional":true,"configurationsDiffer":true},"health":{"score":31.068920105355577,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":20.712613403570383,"upperBound":54.045946736903716,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.7-plus","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":915.27,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":33.53658536585366,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":73,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":33.53658536585366,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 73,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 73.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":42.30769230769231,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":22.4188790560472,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":42.30769230769231,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 22,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 22.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":46.34146341463415,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":17.391220980015497,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":46.34146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 17,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 17.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":55.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":12.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":55.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 12,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 12.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":30.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":767.63,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":95.65217391304348,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":77.9591836734694,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.65217391304348,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 78,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 78.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":57.31707317073171,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.0833333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.31707317073171,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":53.65853658536585,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":35.6348470806302,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":53.65853658536585,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 35,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 35.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":75,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":80.4624277456647,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":75,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 80,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 80.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":43.292682926829265,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:a87fce42-eea3-4e45-a96d-fe057814f371","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.01010101010101,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":49.390243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.7 Plus. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/a87fce42-eea3-4e45-a96d-fe057814f371/qwen3-7-plus/qwen3-7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Model snapshot: Qwen3.7 Plus. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.7 Plus (Release qwen3-7-plus, Zeile qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.7 Plus (release qwen3-7-plus, row qwen3-7-plus, UUID a87fce42-eea3-4e45-a96d-fe057814f371, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":52.809,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":25,"participants":57,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 654.983. Standardfehler: 0.649. Kosten je Aufgabe in USD: 0.095849. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 654.983. Standard error: 0.649. Cost per task in USD: 0.095849. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":20.967741935483872,"measurements":[{"observationId":"vals-code-migration:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":12.856,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20.967741935483872,"participants":63,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2146.993. Standardfehler: 2.931. Kosten je Aufgabe in USD: 0.41923. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 2146.993. Standard error: 2.931. Cost per task in USD: 0.41923. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":28.8135593220339,"measurements":[{"observationId":"vals-emb:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":49.345,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":28.8135593220339,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2209.243. Standardfehler: 3.181. Kosten je Aufgabe in USD: 0.597724. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 2209.243. Standard error: 3.181. Cost per task in USD: 0.597724. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":20,"measurements":[{"observationId":"vals-finance-agent-v2:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":38.22,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":20,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 542.045. Standardfehler: 1.04. Kosten je Aufgabe in USD: 0.357661. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 542.045. Standard error: 1.04. Cost per task in USD: 0.357661. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":10.655737704918034,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":10.655737704918034,"participants":62,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 529.178. Standardfehler: 0. Kosten je Aufgabe in USD: 0.234226. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 529.178. Standard error: 0. Cost per task in USD: 0.234226. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":26.666666666666668,"measurements":[{"observationId":"vals-legal-research:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":16.346,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":26.666666666666668,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 831.995. Standardfehler: 2.57. Kosten je Aufgabe in USD: 0.295949. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 831.995. Standard error: 2.57. Cost per task in USD: 0.295949. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":50,"measurements":[{"observationId":"vals-mortgage-tax:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":66.176,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":53,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 60.548. Standardfehler: 0.928. Kosten je Aufgabe in USD: 0.007588. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 60.548. Standard error: 0.928. Cost per task in USD: 0.007588. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":24.137931034482758,"measurements":[{"observationId":"vals-sage:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":39.251,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.137931034482758,"participants":59,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 177.044. Standardfehler: 3.384. Kosten je Aufgabe in USD: 0.021774. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 177.044. Standard error: 3.384. Cost per task in USD: 0.021774. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":62.06896551724138,"measurements":[{"observationId":"vals-skillsbench:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":54.296,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.06896551724138,"participants":30,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 724.46. Standardfehler: 4.333. Kosten je Aufgabe in USD: 0.137331. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 724.46. Standard error: 4.333. Cost per task in USD: 0.137331. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":21.818181818181817,"measurements":[{"observationId":"vals-tax-agent-bench:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":10.466,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":21.818181818181817,"participants":56,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 310.263. Standardfehler: 1.993. Kosten je Aufgabe in USD: 0.093885. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 310.263. Standard error: 1.993. Cost per task in USD: 0.093885. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":40.27777777777778,"measurements":[{"observationId":"vals-vibe-code-bench:overall:alibaba/qwen3.7-plus","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":46.391,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.27777777777778,"participants":73,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.7-plus. Name des Quellmodells: Qwen 3.7 Plus. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2231.75. Standardfehler: 4.614. Kosten je Aufgabe in USD: 1.082348. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.7-plus. Source model name: Qwen 3.7 Plus. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536. Latency in seconds: 2231.75. Standard error: 4.614. Cost per task in USD: 1.082348. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":21,"totalFamilies":71,"sourceCount":2,"coverage":0.22723922902494334,"lowerBound":9.169536673017086,"upperBound":86.44561377052275,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":3,"totalFamilies":15,"sourceCount":2,"coverage":0.1777777777777778,"lowerBound":6.007153888743191,"upperBound":88.2293761109654,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":16.550522648083625,"upperBound":80.83623693379792,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":19.10569105691057,"upperBound":85.77235772357724,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":5,"totalFamilies":12,"sourceCount":2,"coverage":0.4166666666666667,"lowerBound":19.572038558581873,"upperBound":77.9053718919152,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":19.1304347826087,"upperBound":99.1304347826087,"provisional":true,"configurationsDiffer":true},"finance":{"score":30.157935285053927,"coveredFamilies":4,"totalFamilies":7,"sourceCount":1,"coverage":0.5714285714285714,"lowerBound":17.23310587717367,"upperBound":60.09024873431653,"provisional":true,"configurationsDiffer":true},"legal":{"score":18.661202185792348,"coveredFamilies":2,"totalFamilies":4,"sourceCount":1,"coverage":0.5,"lowerBound":9.330601092896174,"upperBound":59.330601092896174,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":9.375,"upperBound":96.875,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":12.068965517241379,"upperBound":62.06896551724138,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.8-2.4t-a95b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":80.48780487804878,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1435.33,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":80.48780487804878,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":80.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 80,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 80.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":57.247376545822206,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 57,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 57.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":15.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 15,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 15.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1607.97,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":62.19512195121951,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":4.316666666666666,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":62.19512195121951,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":73.17073170731707,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":42.4467099165894,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":73.17073170731707,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 42,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 42.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":65.85365853658537,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:8df710d3-9dae-4498-9b4e-9818238e6f31","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":11.1111111111111,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.85365853658537,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Name des Quellmodells: Qwen 3.8 2.4T A95B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 11,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/8df710d3-9dae-4498-9b4e-9818238e6f31/qwen3-8-2-4t-a95b/qwen3-8-2-4t-a95b. Source model name: Qwen 3.8 2.4T A95B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 2.4T A95B. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 11.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 2.4T A95B (Release qwen3-8-2-4t-a95b, Zeile qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 2.4T A95B (release qwen3-8-2-4t-a95b, row qwen3-8-2-4t-a95b, UUID 8df710d3-9dae-4498-9b4e-9818238e6f31, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":8,"totalFamilies":71,"sourceCount":1,"coverage":0.06955782312925171,"lowerBound":4.844035175045629,"upperBound":97.88825286212045,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":2.195121951219513,"upperBound":98.86178861788619,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":24.564459930313586,"upperBound":88.85017421602788,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":20.73170731707317,"upperBound":87.39837398373984,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":20.32520325203252,"upperBound":95.32520325203252,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.8-27b","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":1395.27,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":81.70731707317073,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":82,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":81.70731707317073,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 82,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 82.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":48.24244368224193,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 48,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 48.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":20,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":34.2324,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":20,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 34,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 34.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":68.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":16.400000000000002,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":68.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 16,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 16.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":67.07317073170732,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":1419.88,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":67.07317073170732,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":45.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":-9.983333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":45.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":67.76315789473684,"measurements":[{"observationId":"vals-gpqa:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":88.889,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.76315789473684,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 116.361. Standardfehler: 1.579. Kosten je Aufgabe in USD: 0.046624. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 116.361. Standard error: 1.579. Cost per task in USD: 0.046624. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":33.920296570899,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 33,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 33.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":52.65957446808511,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":76.3005780346821,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 76,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 76.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":83.931,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.319148936170215,"participants":48,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 43.483. Standardfehler: 0.883. Kosten je Aufgabe in USD: 0.012642. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 43.483. Standard error: 0.883. Cost per task in USD: 0.012642. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":81.35593220338983,"measurements":[{"observationId":"vals-swebench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":86,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.35593220338983,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 599.776. Standardfehler: 1.553. Kosten je Aufgabe in USD: 1.077188. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 599.776. Standard error: 1.553. Cost per task in USD: 1.077188. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":55.99956445993032,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":5.55555555555556,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.97560975609756,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 5,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 5.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":58.427,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":41.07142857142857,"participants":57,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 892.585. Standardfehler: 5.254. Kosten je Aufgabe in USD: 0.677989. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 892.585. Standard error: 5.254. Cost per task in USD: 0.677989. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":10.02805049088359,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:b01dee41-c62b-48ed-8d16-984adc405e5c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=Xhigh","retrievedAt":"2026-10-19","value":0.476190476190476,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":15.217391304347826,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/b01dee41-c62b-48ed-8d16-984adc405e5c/qwen3-8-27b/qwen3-8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 27B (Xhigh). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 27B (Xhigh) (Release qwen3-8-27b, Zeile qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning-Stufe: Xhigh","en":"Exact AA configuration: Qwen3.8 27B (Xhigh) (release qwen3-8-27b, row qwen3-8-27b, UUID b01dee41-c62b-48ed-8d16-984adc405e5c, reasoning=true, source reasoning-token configuration budget=2000, suffix=Xhigh). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Reasoning effort: Xhigh"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":"Xhigh","weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":4.838709677419355,"participants":32,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 14549.775. Standardfehler: 0. Kosten je Aufgabe in USD: 11.135637. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 14549.775. Standard error: 0. Cost per task in USD: 11.135637. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: Mini-SWE-agent. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":27.419354838709676,"measurements":[{"observationId":"vals-code-migration:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":14.157,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.419354838709676,"participants":63,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 17150.223. Standardfehler: 4.189. Kosten je Aufgabe in USD: 21.677954. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 17150.223. Standard error: 4.189. Cost per task in USD: 21.677954. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":51.515151515151516,"measurements":[{"observationId":"vals-cyberbench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":67.798,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":51.515151515151516,"participants":34,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2496.156. Standardfehler: 5.68. Kosten je Aufgabe in USD: 0.990223. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 2496.156. Standard error: 5.68. Cost per task in USD: 0.990223. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":50.847457627118644,"measurements":[{"observationId":"vals-emb:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":59.661,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50.847457627118644,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1370.794. Standardfehler: 3.029. Kosten je Aufgabe in USD: 4.556437. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1370.794. Standard error: 3.029. Cost per task in USD: 4.556437. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":43.333333333333336,"measurements":[{"observationId":"vals-finance-agent-v2:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":48.553,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 621.733. Standardfehler: 2.096. Kosten je Aufgabe in USD: 0.748532. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 621.733. Standard error: 2.096. Cost per task in USD: 0.748532. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":12.121212121212121,"measurements":[{"observationId":"vals-ioi:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":39.056,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":12.121212121212121,"participants":34,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 5924.968. Standardfehler: 4.938. Kosten je Aufgabe in USD: 3.553383. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 5924.968. Standard error: 4.938. Cost per task in USD: 3.553383. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":61.03896103896104,"measurements":[{"observationId":"vals-lcb:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":83.996,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":61.03896103896104,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 149.298. Standardfehler: 1.035. Kosten je Aufgabe in USD: 0.044983. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 149.298. Standard error: 1.035. Cost per task in USD: 0.044983. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":86.06557377049181,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":11.25,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":86.06557377049181,"participants":62,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1118.634. Standardfehler: 2.415. Kosten je Aufgabe in USD: 4.033621. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1118.634. Standard error: 2.415. Cost per task in USD: 4.033621. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":43.58974358974359,"measurements":[{"observationId":"vals-legal-bench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":82.428,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":43.58974358974359,"participants":79,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 47.806. Standardfehler: 0.474. Kosten je Aufgabe in USD: 0.004666. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 47.806. Standard error: 0.474. Cost per task in USD: 0.004666. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":53.333333333333336,"measurements":[{"observationId":"vals-legal-research:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":36.058,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":53.333333333333336,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1600.721. Standardfehler: 3.337. Kosten je Aufgabe in USD: 2.159719. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1600.721. Standard error: 3.337. Cost per task in USD: 2.159719. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":2.985074626865672,"measurements":[{"observationId":"vals-medcode:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":28.698,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":2.985074626865672,"participants":68,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 89.379. Standardfehler: 1.971. Kosten je Aufgabe in USD: 0.050145. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 89.379. Standard error: 1.971. Cost per task in USD: 0.050145. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":58.8235294117647,"measurements":[{"observationId":"vals-medscribe:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":83.849,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.8235294117647,"participants":69,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 89.705. Standardfehler: 1.977. Kosten je Aufgabe in USD: 0.046732. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 89.705. Standard error: 1.977. Cost per task in USD: 0.046732. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":42.857142857142854,"measurements":[{"observationId":"vals-mmlu-pro:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":84.337,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":42.857142857142854,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 69.848. Standardfehler: 0.356. Kosten je Aufgabe in USD: 0.010543. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 69.848. Standard error: 0.356. Cost per task in USD: 0.010543. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":36.53846153846154,"measurements":[{"observationId":"vals-mortgage-tax:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":64.944,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":36.53846153846154,"participants":53,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 79.952. Standardfehler: 0.947. Kosten je Aufgabe in USD: 0.011604. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 79.952. Standard error: 0.947. Cost per task in USD: 0.011604. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 21400.403. Standardfehler: 0. Kosten je Aufgabe in USD: 17.990872. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 21400.403. Standard error: 0. Cost per task in USD: 17.990872. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":17.94871794871795,"measurements":[{"observationId":"vals-proof-bench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":16,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":17.94871794871795,"participants":40,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1453.872. Standardfehler: 3.685. Kosten je Aufgabe in USD: 2.298832. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1453.872. Standard error: 3.685. Cost per task in USD: 2.298832. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":91.37931034482759,"measurements":[{"observationId":"vals-sage:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":52.401,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":91.37931034482759,"participants":59,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 83.03. Standardfehler: 3.467. Kosten je Aufgabe in USD: 0.035817. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000. Latency in seconds: 83.03. Standard error: 3.467. Cost per task in USD: 0.035817. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 30000","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":13.793103448275861,"measurements":[{"observationId":"vals-skillsbench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":38.106,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":13.793103448275861,"participants":30,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 401.009. Standardfehler: 4.38. Kosten je Aufgabe in USD: 0.81809. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 401.009. Standard error: 4.38. Cost per task in USD: 0.81809. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":69.0909090909091,"measurements":[{"observationId":"vals-tax-agent-bench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":30.894,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.0909090909091,"participants":56,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 1333.825. Standardfehler: 3.323. Kosten je Aufgabe in USD: 0.81533. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 1333.825. Standard error: 3.323. Cost per task in USD: 0.81533. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":27.63157894736842,"measurements":[{"observationId":"vals-tax-eval-v2:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":70.85,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":27.63157894736842,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 94.491. Standardfehler: 0.893. Kosten je Aufgabe in USD: 0.046465. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 94.491. Standard error: 0.893. Cost per task in USD: 0.046465. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":58.333333333333336,"measurements":[{"observationId":"vals-vibe-code-bench:overall:alibaba/qwen3.8-27b","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":64.85,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":58.333333333333336,"participants":73,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-27b. Name des Quellmodells: Qwen 3.8 27B. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536. Latenz in Sekunden: 2619.407. Standardfehler: 4.825. Kosten je Aufgabe in USD: 23.456372. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-27b. Source model name: Qwen 3.8 27B. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536. Latency in seconds: 2619.407. Standard error: 4.825. Cost per task in USD: 23.456372. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Reasoning-Stufe: xhigh. Temperatur: 1. Top P: 0.95. Maximale Ausgabetoken: 65536","en":"Harness: OpenHands. Reasoning effort: xhigh. Temperature: 1. Top P: 0.95. Maximum output tokens: 65536"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":"xhigh","weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":33,"totalFamilies":71,"sourceCount":2,"coverage":0.43242630385487535,"lowerBound":22.159252997193853,"upperBound":78.91662261170632,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":35.686740834667205,"coveredFamilies":9,"totalFamilies":15,"sourceCount":2,"coverage":0.5444444444444446,"lowerBound":19.429447787763262,"upperBound":64.9850033433188,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":29.84022556390977,"upperBound":86.98308270676691,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":3.5897435897435894,"upperBound":83.58974358974359,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":43.98954703832753,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":29.326364692218352,"upperBound":62.65969802555169,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":4,"totalFamilies":12,"sourceCount":2,"coverage":0.3333333333333333,"lowerBound":18.832352116624612,"upperBound":85.49901878329129,"provisional":true,"configurationsDiffer":true},"finance":{"score":45.48834810743821,"coveredFamilies":5,"totalFamilies":7,"sourceCount":1,"coverage":0.7142857142857142,"lowerBound":32.49167721959872,"upperBound":61.0631057910273,"provisional":true,"configurationsDiffer":true},"legal":{"score":60.99621689785624,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":45.74716267339218,"upperBound":70.74716267339218,"provisional":true,"configurationsDiffer":true},"health":{"score":30.90430201931519,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":20.602868012876794,"upperBound":53.936201346210126,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":51.515151515151516,"upperBound":51.515151515151516,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":13.164893617021278,"upperBound":88.16489361702128,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":45.689655172413794,"upperBound":95.6896551724138,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen-3.8-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"qwen-3.8-max","contributions":[{"familyId":"artificial-analysis-aa-analyst-agent","area":"agentic","label":{"de":"AA-AnalystAgent","en":"AA-AnalystAgent"},"weight":0.005952380952380952,"coverage":1,"score":60.526315789473685,"measurements":[{"observationId":"aa:artificial-analysis-aa-analyst-agent:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","benchmarkId":"artificial-analysis-aa-analyst-agent","benchmarkVersion":"AA cohort: 80 questions","comparisonCohort":"artificial-analysis-aa-analyst-agent:AA cohort: 80 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":45,"unit":"%","metric":"pass^5 reliability","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":60.526315789473685,"participants":39,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Angezeigter Wert: 45,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled. Score display: 45.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n80 quantitative Fragen aus 14 Domänen werden je fünfmal unabhängig ausgeführt. pass^5 zählt nur Fragen als gelöst, die in allen fünf Versuchen richtig beantwortet werden.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Werkzeugrichtlinie: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Versuche: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n80 quantitative questions across 14 domains are run five independent times. pass^5 counts only questions answered correctly in all five attempts.. Harness: Artificial Analysis Stirrup. Agent: Stirrup analyst agent. Tool policy: 100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission. Attempts: pass^5, 5 independent attempts per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-analyst-agent\",\"AA cohort: 80 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup analyst agent","toolPolicy":"100-turn cap; Python sandbox with source files, URL fetch, image viewing, and answer submission","attempts":"pass^5, 5 independent attempts per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":95.1219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1612.8,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":95.1219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":80.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 80,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 80.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":96.15384615384616,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":42.4041297935103,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":96.15384615384616,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 42,4 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 42.4%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":74.39024390243902,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":56.1905381087678,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":74.39024390243902,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 56,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 56.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-frontiercode","area":"coding","label":{"de":"FrontierCode 1.1 Main","en":"FrontierCode 1.1 Main"},"weight":0.004761904761904762,"coverage":1,"score":66.66666666666667,"measurements":[{"observationId":"aa:artificial-analysis-frontiercode:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode","benchmarkId":"artificial-analysis-frontiercode","benchmarkVersion":"1.1 Main","comparisonCohort":"artificial-analysis-frontiercode:1.1 Main:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":43.156299999999995,"unit":"%","metric":"Mean score over three attempts per task","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":66.66666666666667,"participants":16,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Angezeigter Wert: 43,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled. Score display: 43.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie primären 100 Aufgaben aus FrontierCode 1.1 werden je dreimal ausgeführt. Alle Aufgaben zählen gleich. Das Verfehlen einer blockierenden Prüfung oder ein Treffer des Trajectory-Scanners ergibt null; Läufe, die alle blockierenden Prüfungen bestehen, erhalten 70 bis 100 Prozent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Werkzeugrichtlinie: 250-turn cap; repository worktree graded as left by the agent. Versuche: Mean score, 3 attempts per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe primary 100-task FrontierCode 1.1 set is run three times per task, with equal task weights. Failing any blocking check or being flagged by the trajectory scanner scores zero; runs passing every blocking check score 70 to 100 percent.. Harness: Artificial Analysis Stirrup. Agent: Stirrup coding agent. Tool policy: 250-turn cap; repository worktree graded as left by the agent. Attempts: Mean score, 3 attempts per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-frontiercode\",\"1.1 Main\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup coding agent","toolPolicy":"250-turn cap; repository worktree graded as left by the agent","attempts":"Mean score, 3 attempts per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":85.36585365853658,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":22.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":85.36585365853658,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 22,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 22.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":93.90243902439025,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1665.58,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":93.90243902439025,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":50,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":40.254237288135606,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 40,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 40.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":69.51219512195122,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":11.983333333333333,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.51219512195122,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"gpqa-diamond","area":"reasoning","label":{"de":"GPQA Diamond","en":"GPQA Diamond"},"weight":0.01020408163265306,"coverage":0.5,"score":93.42105263157895,"measurements":[{"observationId":"vals-gpqa:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","benchmarkId":"vals-gpqa","benchmarkVersion":"1","comparisonCohort":"vals-gpqa:1:overall","retrievedAt":"2026-10-15","value":93.686,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":93.42105263157895,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 266.606. Standardfehler: 1.221. Kosten je Aufgabe in USD: 0.077028. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-gpqa-2026-10-15-951eb5224097. Prüfsumme der Quelldaten: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 266.606. Standard error: 1.221. Cost per task in USD: 0.077028. Source ID: vals-ai. Evidence capture ID: vals-gpqa-2026-10-15-951eb5224097. Source payload hash: 951eb5224097733a9d09d06c14908ae2980e2e640e539f2c284877c5092d7077. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-gpqa:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":78.04878048780488,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":43.0954587581094,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":78.04878048780488,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 43,1 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 43.1%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"lmarena-code-webdev","area":"coding","label":{"de":"LMArena Code Arena WebDev","en":"LMArena Code Arena WebDev"},"weight":0.004761904761904762,"coverage":1,"score":80.55555555555556,"measurements":[{"observationId":"lmarena:webdev:overall:qwen3.8-max-0902","sourceId":"lmarena","sourceName":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/tree/4589dee83b594891961ae5190c508a9288d2c2ae/webdev","benchmarkId":"lmarena-code-webdev","benchmarkVersion":"webdev","comparisonCohort":"lmarena-code-webdev:2026-10-13:overall","retrievedAt":"2026-10-15","value":1673.1508314042985,"unit":"rating","metric":"WebDev Arena Bradley-Terry rating","task":"overall","evaluator":"LMArena","higherIsBetter":true,"percentile":80.55555555555556,"participants":37,"methodology":{"de":"Arena-Bewertung aus 13.081 veröffentlichten Paarvergleichen.. Kennung des Quellmodells: lmarena/qwen3.8-max-0902. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: qwen3.8-max-0902. Evaluator: LMArena. Datensatztyp: public-crowdsourced-preferences. Stand der Rangliste: 2026-10-13. Harness: Arena Code Arena WebDev. Quellenkennung: lmarena. Kennung der Evidenzaufnahme: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Prüfsumme der Quelldaten: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parserversion: lmarena-dataset-v1. Importiert am: 2026-10-15","en":"Arena rating from 13,081 published pairwise votes.. Source model ID: lmarena/qwen3.8-max-0902. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: qwen3.8-max-0902. Evaluator: LMArena. Dataset type: public-crowdsourced-preferences. Leaderboard date: 2026-10-13. Harness: Arena Code Arena WebDev. Source ID: lmarena. Evidence capture ID: lmarena-webdev-4589dee83b594891961ae5190c508a9288d2c2ae-28f8076e6c27. Source payload hash: 28f8076e6c27f1f89239f3d7243456357dfa175b078646a6bd859935b162cc9c. Parser version: lmarena-dataset-v1. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Arena-Bewertung aus 13.081 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 13,081 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreRunConfiguration":{"de":"Arena-Bewertung aus 13.081 veröffentlichten Paarvergleichen.. Harness: Arena Code Arena WebDev","en":"Arena rating from 13,081 published pairwise votes.. Harness: Arena Code Arena WebDev"},"scoreComparisonSetup":{"publicationCohort":"lmarena-code-webdev:2026-10-13:overall","datasetType":"public-crowdsourced-preferences","harness":"Arena Code Arena WebDev","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":1,"score":80.26215805471125,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":82.7745664739884,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":83.92857142857143,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 82,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 82.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-mmmu:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","benchmarkId":"vals-mmmu","benchmarkVersion":"1","comparisonCohort":"vals-mmmu:1:overall","retrievedAt":"2026-10-15","value":88.035,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.59574468085107,"participants":48,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 43.853. Standardfehler: 0.78. Kosten je Aufgabe in USD: 0.015881. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmmu-2026-10-15-29395186eddd. Prüfsumme der Quelldaten: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 43.853. Standard error: 0.78. Cost per task in USD: 0.015881. Source ID: vals-ai. Evidence capture ID: vals-mmmu-2026-10-15-29395186eddd. Source payload hash: 29395186eddd2c70601228c584a14b3c1d8f9dba4b68256561b46b0dd03c635a. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmmu:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"swe-bench-verified","area":"coding","label":{"de":"SWE-bench Verified","en":"SWE-bench Verified"},"weight":0.004761904761904762,"coverage":0.5,"score":79.66101694915254,"measurements":[{"observationId":"vals-swebench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","benchmarkId":"vals-swebench","benchmarkVersion":"1","comparisonCohort":"vals-swebench:1:overall","retrievedAt":"2026-10-15","value":85.6,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":79.66101694915254,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2502.097. Standardfehler: 1.572. Kosten je Aufgabe in USD: 1.128442. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-2026-10-15-7a4e2e848528. Prüfsumme der Quelldaten: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 2502.097. Standard error: 1.572. Cost per task in USD: 1.128442. Source ID: vals-ai. Evidence capture ID: vals-swebench-2026-10-15-7a4e2e848528. Source payload hash: 7a4e2e8485283caac500127419f68ec4bd5414c837ade114fc6afa5b2d34ccaa. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.8333333333333333,"score":75.8470665787739,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":38.8888888888889,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":89.02439024390245,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 38,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 38.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2-1:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","benchmarkId":"vals-terminal-bench-2-1","benchmarkVersion":"2.1","comparisonCohort":"vals-terminal-bench-2-1:2.1:overall","retrievedAt":"2026-10-15","value":67.416,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":57,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-28. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 890.917. Standardfehler: 0.649. Kosten je Aufgabe in USD: 0.448084. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Prüfsumme der Quelldaten: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-28. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 890.917. Standard error: 0.649. Cost per task in USD: 0.448084. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-1-2026-10-15-a5306805cdf8. Source payload hash: a5306805cdf86495bdcceff6b2c8534a3768b9853f5077895dab975b809b054f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2-1:2.1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-4:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","benchmarkId":"vals-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"vals-terminal-bench-4:4.0:overall","retrievedAt":"2026-10-15","value":34.343,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":62.16216216216216,"participants":38,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 8900.152. Standardfehler: 3.945. Kosten je Aufgabe in USD: 10.633329. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Prüfsumme der Quelldaten: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 8900.152. Standard error: 3.945. Cost per task in USD: 10.633329. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-4-2026-10-15-c1ed9f87952d. Source payload hash: c1ed9f87952d6458c39e925b798cd9f8809261271e108b6580e062edcc4b6528. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-4:4.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"terminal-bench-science","area":"coding","label":{"de":"Terminal-Bench Science","en":"Terminal-Bench Science"},"weight":0.004761904761904762,"coverage":1,"score":64.86676016830295,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-science:b112af07-3bd5-4647-b09f-b23204361bb2","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science","benchmarkId":"artificial-analysis-terminal-bench-science","benchmarkVersion":"0.1.0","comparisonCohort":"artificial-analysis-terminal-bench-science:0.1.0:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":11.9047619047619,"unit":"%","metric":"pass@1 task success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":65.21739130434783,"participants":24,"methodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Kennung des Quellmodells: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Modellstand: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 11,9 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Source model ID: artificial-analysis/b112af07-3bd5-4647-b09f-b23204361bb2/qwen3-8-max-0902/qwen3-8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Model snapshot: Qwen3.8 Max (0902). Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 11.9%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Qwen3.8 Max (0902) (Release qwen3-8-max-0902, Zeile qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDie vollständige Version 0.1.0 umfasst 70 Aufgaben aus fünf Wissenschaftsbereichen. Jede Aufgabe besteht nur, wenn ihre gesamte Testsuite besteht. AA mittelt pass@1 über drei Wiederholungen.. Harness: mini-swe-agent. Agent: mini-swe-agent. Werkzeugrichtlinie: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Qwen3.8 Max (0902) (release qwen3-8-max-0902, row qwen3-8-max, UUID b112af07-3bd5-4647-b09f-b23204361bb2, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe full 0.1.0 release has 70 tasks across five scientific domains. A task passes only when its entire test suite passes. AA averages pass@1 over three repeats.. Harness: mini-swe-agent. Agent: mini-swe-agent. Tool policy: 1,000-step cap; task timeouts and sandbox resources follow task definitions. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-science\",\"0.1.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"mini-swe-agent","agent":"mini-swe-agent","toolPolicy":"1,000-step cap; task timeouts and sandbox resources follow task definitions","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-science:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","benchmarkId":"vals-terminal-bench-science","benchmarkVersion":"0.1","comparisonCohort":"vals-terminal-bench-science:0.1:overall","retrievedAt":"2026-10-15","value":12.857,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":64.51612903225806,"participants":32,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Harness: Mini-SWE-agent. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 12243.338. Standardfehler: 4.03. Kosten je Aufgabe in USD: 16.682189. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Prüfsumme der Quelldaten: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Harness: Mini-SWE-agent. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 12243.338. Standard error: 4.03. Cost per task in USD: 16.682189. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-science-2026-10-15-8b1e74f0d5cc. Source payload hash: 8b1e74f0d5cc2830c74ef940b332180d3404ec4cc23726b25621fc186183e485. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: Mini-SWE-agent. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: Mini-SWE-agent. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: Mini-SWE-agent. Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-science:0.1:overall","datasetType":"public","harness":"Mini-SWE-agent","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"vals-code-migration","area":"coding","label":{"de":"Code Migration","en":"Code Migration"},"weight":0.004761904761904762,"coverage":1,"score":40.32258064516129,"measurements":[{"observationId":"vals-code-migration:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","benchmarkId":"vals-code-migration","benchmarkVersion":"1","comparisonCohort":"vals-code-migration:1:overall","retrievedAt":"2026-10-15","value":23.958,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":40.32258064516129,"participants":63,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 16065.728. Standardfehler: 4.309. Kosten je Aufgabe in USD: 10.53816. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-code-migration-2026-10-15-3fa9c10568ec. Prüfsumme der Quelldaten: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 16065.728. Standard error: 4.309. Cost per task in USD: 10.53816. Source ID: vals-ai. Evidence capture ID: vals-code-migration-2026-10-15-3fa9c10568ec. Source payload hash: 3fa9c10568ecc117fabec81767c405b49a25d5d2ded308ee435fffb197c53474. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-code-migration:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":77.02702702702703,"measurements":[{"observationId":"vals-corp-fin-v2:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":65.851,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":77.02702702702703,"participants":75,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 46.649. Standardfehler: 0.935. Kosten je Aufgabe in USD: 0.179323. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 46.649. Standard error: 0.935. Cost per task in USD: 0.179323. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-cyberbench","area":"cybersecurity","label":{"de":"CyberBench v1.1","en":"CyberBench v1.1"},"weight":0.07142857142857142,"coverage":1,"score":0,"measurements":[{"observationId":"vals-cyberbench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","benchmarkId":"vals-cyberbench","benchmarkVersion":"1.1","comparisonCohort":"vals-cyberbench:1.1:overall","retrievedAt":"2026-10-15","value":28.572,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":0,"participants":34,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 580.168. Standardfehler: 3.306. Kosten je Aufgabe in USD: 0.784693. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-cyber-2026-10-15-943f593f929f. Prüfsumme der Quelldaten: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 580.168. Standard error: 3.306. Cost per task in USD: 0.784693. Source ID: vals-ai. Evidence capture ID: vals-cyber-2026-10-15-943f593f929f. Source payload hash: 943f593f929f8971a6b746269a2851b5febc2aad4e2f1066895f715033cdc7f1. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-cyberbench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-emb","area":"finance","label":{"de":"EMB","en":"EMB"},"weight":0.01020408163265306,"coverage":1,"score":52.54237288135593,"measurements":[{"observationId":"vals-emb:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","benchmarkId":"vals-emb","benchmarkVersion":"1","comparisonCohort":"vals-emb:1:overall","retrievedAt":"2026-10-15","value":60.066,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":52.54237288135593,"participants":60,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3467.209. Standardfehler: 3.352. Kosten je Aufgabe in USD: 2.655924. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-emb-2026-10-15-e1170aaade72. Prüfsumme der Quelldaten: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 3467.209. Standard error: 3.352. Cost per task in USD: 2.655924. Source ID: vals-ai. Evidence capture ID: vals-emb-2026-10-15-e1170aaade72. Source payload hash: e1170aaade72d6b81079f9cb90244c41f028a12e29d982c3ec1d6e9913a9872d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-emb:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-finance-agent-v2","area":"finance","label":{"de":"Finance Agent (v2)","en":"Finance Agent (v2)"},"weight":0.01020408163265306,"coverage":1,"score":50,"measurements":[{"observationId":"vals-finance-agent-v2:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","benchmarkId":"vals-finance-agent-v2","benchmarkVersion":"2","comparisonCohort":"vals-finance-agent-v2:2:overall","retrievedAt":"2026-10-15","value":50.593,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":50,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1174.928. Standardfehler: 0.578. Kosten je Aufgabe in USD: 1.236977. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-fabv2-2026-10-15-3780be1a23c9. Prüfsumme der Quelldaten: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 1174.928. Standard error: 0.578. Cost per task in USD: 1.236977. Source ID: vals-ai. Evidence capture ID: vals-fabv2-2026-10-15-3780be1a23c9. Source payload hash: 3780be1a23c9133a3c34a6095024af162b35a8ef50963a62fc67a6049880426c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-finance-agent-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-ioi","area":"coding","label":{"de":"IOI","en":"IOI"},"weight":0.004761904761904762,"coverage":1,"score":69.6969696969697,"measurements":[{"observationId":"vals-ioi:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","benchmarkId":"vals-ioi","benchmarkVersion":"2","comparisonCohort":"vals-ioi:2:overall","retrievedAt":"2026-10-15","value":68.889,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":69.6969696969697,"participants":34,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-14. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 6398.213. Standardfehler: 5.204. Kosten je Aufgabe in USD: 9.468852. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-ioi-2026-10-15-e1545b21b5b5. Prüfsumme der Quelldaten: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-14. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 6398.213. Standard error: 5.204. Cost per task in USD: 9.468852. Source ID: vals-ai. Evidence capture ID: vals-ioi-2026-10-15-e1545b21b5b5. Source payload hash: e1545b21b5b5c98d037aefa9d441b65859d07a10550f242d0e16162a49b8e8ab. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-ioi:2:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-lcb","area":"coding","label":{"de":"LiveCodeBench v6","en":"LiveCodeBench v6"},"weight":0.004761904761904762,"coverage":1,"score":89.6103896103896,"measurements":[{"observationId":"vals-lcb:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","benchmarkId":"vals-lcb","benchmarkVersion":"1","comparisonCohort":"vals-lcb:1:overall","retrievedAt":"2026-10-15","value":87.852,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":89.6103896103896,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 337.788. Standardfehler: 0.951. Kosten je Aufgabe in USD: 0.091412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-lcb-2026-10-15-1a7c7dabc9d9. Prüfsumme der Quelldaten: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 337.788. Standard error: 0.951. Cost per task in USD: 0.091412. Source ID: vals-ai. Evidence capture ID: vals-lcb-2026-10-15-1a7c7dabc9d9. Source payload hash: 1a7c7dabc9d95a0182886b1edc42623987cac2095a0d407a5e334f37acdfa2f5. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-lcb:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-agent-benchmark","area":"legal","label":{"de":"Harvey's Legal Agent Benchmark","en":"Harvey's Legal Agent Benchmark"},"weight":0.017857142857142856,"coverage":1,"score":83.60655737704919,"measurements":[{"observationId":"vals-legal-agent-benchmark:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","benchmarkId":"vals-legal-agent-benchmark","benchmarkVersion":"1","comparisonCohort":"vals-legal-agent-benchmark:1:overall","retrievedAt":"2026-10-15","value":10.417,"unit":"%","metric":"task resolution rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.60655737704919,"participants":62,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2952.433. Standardfehler: 2.287. Kosten je Aufgabe in USD: 2.365468. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-hlab-2026-10-15-7587eca48586. Prüfsumme der Quelldaten: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 2952.433. Standard error: 2.287. Cost per task in USD: 2.365468. Source ID: vals-ai. Evidence capture ID: vals-hlab-2026-10-15-7587eca48586. Source payload hash: 7587eca48586be017c9db8c1a11e6f6797b1eb62b7627cbf198f35f592f4fc69. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-agent-benchmark:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-bench","area":"legal","label":{"de":"LegalBench","en":"LegalBench"},"weight":0.017857142857142856,"coverage":1,"score":56.41025641025641,"measurements":[{"observationId":"vals-legal-bench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","benchmarkId":"vals-legal-bench","benchmarkVersion":"1","comparisonCohort":"vals-legal-bench:1:overall","retrievedAt":"2026-10-15","value":83.609,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.41025641025641,"participants":79,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 36.373. Standardfehler: 0.418. Kosten je Aufgabe in USD: 0.010412. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_bench-2026-10-15-c8486ba756ba. Prüfsumme der Quelldaten: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 36.373. Standard error: 0.418. Cost per task in USD: 0.010412. Source ID: vals-ai. Evidence capture ID: vals-legal_bench-2026-10-15-c8486ba756ba. Source payload hash: c8486ba756baa5cd3955d035edff67440e8db23e69d62ad1a9e18d15b8c4bc52. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-bench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-legal-research","area":"legal","label":{"de":"Legal Research Bench","en":"Legal Research Bench"},"weight":0.017857142857142856,"coverage":1,"score":83.33333333333333,"measurements":[{"observationId":"vals-legal-research:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","benchmarkId":"vals-legal-research","benchmarkVersion":"1","comparisonCohort":"vals-legal-research:1:overall","retrievedAt":"2026-10-15","value":47.596,"unit":"%","metric":"all-pass rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":83.33333333333333,"participants":61,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 3677.636. Standardfehler: 3.471. Kosten je Aufgabe in USD: 2.486351. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-legal_research-2026-10-15-f16d6b6678c0. Prüfsumme der Quelldaten: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 3677.636. Standard error: 3.471. Cost per task in USD: 2.486351. Source ID: vals-ai. Evidence capture ID: vals-legal_research-2026-10-15-f16d6b6678c0. Source payload hash: f16d6b6678c06111a5c628248d14f0eb5c918491d50288c7f649729fcd5dcfaf. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-legal-research:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medcode","area":"health","label":{"de":"MedCode","en":"MedCode"},"weight":0.023809523809523808,"coverage":1,"score":32.83582089552239,"measurements":[{"observationId":"vals-medcode:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","benchmarkId":"vals-medcode","benchmarkVersion":"1","comparisonCohort":"vals-medcode:1:overall","retrievedAt":"2026-10-15","value":40.668,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":32.83582089552239,"participants":68,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 378.022. Standardfehler: 2.029. Kosten je Aufgabe in USD: 0.120827. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medcode-2026-10-15-988a6ae1e07e. Prüfsumme der Quelldaten: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 378.022. Standard error: 2.029. Cost per task in USD: 0.120827. Source ID: vals-ai. Evidence capture ID: vals-medcode-2026-10-15-988a6ae1e07e. Source payload hash: 988a6ae1e07e33ce031572ed4a804be11c36ee97a1dd75cbe4740fff3b534f8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medcode:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-medscribe","area":"health","label":{"de":"MedScribe","en":"MedScribe"},"weight":0.023809523809523808,"coverage":1,"score":67.6470588235294,"measurements":[{"observationId":"vals-medscribe:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","benchmarkId":"vals-medscribe","benchmarkVersion":"1","comparisonCohort":"vals-medscribe:1:overall","retrievedAt":"2026-10-15","value":84.947,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":67.6470588235294,"participants":69,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-13. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 251.26. Standardfehler: 1.999. Kosten je Aufgabe in USD: 0.089616. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-medscribe-2026-10-15-271fc122ddc2. Prüfsumme der Quelldaten: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-13. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 251.26. Standard error: 1.999. Cost per task in USD: 0.089616. Source ID: vals-ai. Evidence capture ID: vals-medscribe-2026-10-15-271fc122ddc2. Source payload hash: 271fc122ddc2b0602bd6c7947f65a9a7f0d8cab94f6dc27d62a9380a9317f9c0. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-medscribe:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mmlu-pro","area":"knowledge","label":{"de":"MMLU-Pro","en":"MMLU-Pro"},"weight":0.023809523809523808,"coverage":1,"score":81.81818181818181,"measurements":[{"observationId":"vals-mmlu-pro:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","benchmarkId":"vals-mmlu-pro","benchmarkVersion":"1","comparisonCohort":"vals-mmlu-pro:1:overall","retrievedAt":"2026-10-15","value":88.602,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":81.81818181818181,"participants":78,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 63.576. Standardfehler: 0.313. Kosten je Aufgabe in USD: 0.018585. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mmlu_pro-2026-10-15-dab025507f46. Prüfsumme der Quelldaten: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 63.576. Standard error: 0.313. Cost per task in USD: 0.018585. Source ID: vals-ai. Evidence capture ID: vals-mmlu_pro-2026-10-15-dab025507f46. Source payload hash: dab025507f46c716dd8d145e90ec0a737d9c56219853f41e4b1cc3c5e8c222b4. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mmlu-pro:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mortgage-tax","area":"finance","label":{"de":"MortgageTax","en":"MortgageTax"},"weight":0.01020408163265306,"coverage":1,"score":29.807692307692307,"measurements":[{"observationId":"vals-mortgage-tax:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","benchmarkId":"vals-mortgage-tax","benchmarkVersion":"1","comparisonCohort":"vals-mortgage-tax:1:overall","retrievedAt":"2026-10-15","value":63.99,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":29.807692307692307,"participants":53,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 47.499. Standardfehler: 0.942. Kosten je Aufgabe in USD: 0.027349. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mortgage_tax-2026-10-15-e5c844561f97. Prüfsumme der Quelldaten: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 47.499. Standard error: 0.942. Cost per task in USD: 0.027349. Source ID: vals-ai. Evidence capture ID: vals-mortgage_tax-2026-10-15-e5c844561f97. Source payload hash: e5c844561f97d119d66d70269e726ecfb68be79eae39acf14ead3e95422e648c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mortgage-tax:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-mysterymechanism","area":"general","label":{"de":"MysteryMechanism","en":"MysteryMechanism"},"weight":0.014285714285714285,"coverage":1,"score":30,"measurements":[{"observationId":"vals-mysterymechanism:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","benchmarkId":"vals-mysterymechanism","benchmarkVersion":"1","comparisonCohort":"vals-mysterymechanism:1:overall","retrievedAt":"2026-10-15","value":23.874,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":30,"participants":21,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2268.77. Standardfehler: 2.868. Kosten je Aufgabe in USD: 1.309463. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Prüfsumme der Quelldaten: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 2268.77. Standard error: 2.868. Cost per task in USD: 1.309463. Source ID: vals-ai. Evidence capture ID: vals-mysterymechanism-2026-10-15-1ef09cd0a0ee. Source payload hash: 1ef09cd0a0ee8690b0c07b2272057461242359b1680e24975ea42971e03acea9. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-mysterymechanism:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-programbench","area":"coding","label":{"de":"ProgramBench","en":"ProgramBench"},"weight":0.004761904761904762,"coverage":1,"score":24.489795918367346,"measurements":[{"observationId":"vals-programbench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","benchmarkId":"vals-programbench","benchmarkVersion":"1","comparisonCohort":"vals-programbench:1:overall","retrievedAt":"2026-10-15","value":0,"unit":"%","metric":"fully resolved rate","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.489795918367346,"participants":50,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 21263.689. Standardfehler: 0. Kosten je Aufgabe in USD: 12.357107. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-programbench-2026-10-15-e9bbb4378c7a. Prüfsumme der Quelldaten: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 21263.689. Standard error: 0. Cost per task in USD: 12.357107. Source ID: vals-ai. Evidence capture ID: vals-programbench-2026-10-15-e9bbb4378c7a. Source payload hash: e9bbb4378c7a058441a21d456baf9316b96c2cf5b58d371a3c7c803f5000bd8c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-programbench:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-proof-bench","area":"math","label":{"de":"ProofBench","en":"ProofBench"},"weight":0.014285714285714285,"coverage":1,"score":55.12820512820513,"measurements":[{"observationId":"vals-proof-bench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","benchmarkId":"vals-proof-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-proof-bench:1.1:overall","retrievedAt":"2026-10-15","value":58,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":55.12820512820513,"participants":40,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1867.622. Standardfehler: 4.96. Kosten je Aufgabe in USD: 1.503984. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-proof_bench-2026-10-15-22be2aca25de. Prüfsumme der Quelldaten: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 1867.622. Standard error: 4.96. Cost per task in USD: 1.503984. Source ID: vals-ai. Evidence capture ID: vals-proof_bench-2026-10-15-22be2aca25de. Source payload hash: 22be2aca25de5c4a9eb6448a95713e09a7c97126dfc6f4f75c4afd0e563209cb. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-proof-bench:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-sage","area":"education","label":{"de":"SAGE","en":"SAGE"},"weight":0.03571428571428571,"coverage":1,"score":82.75862068965517,"measurements":[{"observationId":"vals-sage:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","benchmarkId":"vals-sage","benchmarkVersion":"1","comparisonCohort":"vals-sage:1:overall","retrievedAt":"2026-10-15","value":51.254,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":82.75862068965517,"participants":59,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-08. Temperatur: 1. Maximale Ausgabetoken: 30000. Latenz in Sekunden: 132.802. Standardfehler: 3.415. Kosten je Aufgabe in USD: 0.073645. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-sage-2026-10-15-cc5fb3fecdb0. Prüfsumme der Quelldaten: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-08. Temperature: 1. Maximum output tokens: 30000. Latency in seconds: 132.802. Standard error: 3.415. Cost per task in USD: 0.073645. Source ID: vals-ai. Evidence capture ID: vals-sage-2026-10-15-cc5fb3fecdb0. Source payload hash: cc5fb3fecdb0382d492ff167a42ad84b9c5c3883373c78ed87baaa972b8b6f7c. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreRunConfiguration":{"de":"Temperatur: 1. Maximale Ausgabetoken: 30000","en":"Temperature: 1. Maximum output tokens: 30000"},"scoreComparisonSetup":{"publicationCohort":"vals-sage:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-skillsbench","area":"agentic","label":{"de":"SkillsBench","en":"SkillsBench"},"weight":0.005952380952380952,"coverage":1,"score":24.137931034482758,"measurements":[{"observationId":"vals-skillsbench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","benchmarkId":"vals-skillsbench","benchmarkVersion":"1","comparisonCohort":"vals-skillsbench:1:overall","retrievedAt":"2026-10-15","value":42.011,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":24.137931034482758,"participants":30,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-09-27. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 948.433. Standardfehler: 4.435. Kosten je Aufgabe in USD: 0.524988. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-skillsbench-2026-10-15-aaad242949d8. Prüfsumme der Quelldaten: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-09-27. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 948.433. Standard error: 4.435. Cost per task in USD: 0.524988. Source ID: vals-ai. Evidence capture ID: vals-skillsbench-2026-10-15-aaad242949d8. Source payload hash: aaad242949d81b7c7c61c36d4895d4460e9146760c1e6b4b38d235a16043bc61. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-skillsbench:1:overall","datasetType":"public","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-snap","area":"productivity","label":{"de":"Public Benefits Bench v1.1","en":"Public Benefits Bench v1.1"},"weight":0.03571428571428571,"coverage":1,"score":73.6842105263158,"measurements":[{"observationId":"vals-snap:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","benchmarkId":"vals-snap","benchmarkVersion":"1.1","comparisonCohort":"vals-snap:1.1:overall","retrievedAt":"2026-10-15","value":67.118,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":73.6842105263158,"participants":39,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2443.976. Standardfehler: 1.222. Kosten je Aufgabe in USD: 0.991386. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-public-benefits-bench-2026-10-15-6f091831ab60. Prüfsumme der Quelldaten: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 2443.976. Standard error: 1.222. Cost per task in USD: 0.991386. Source ID: vals-ai. Evidence capture ID: vals-public-benefits-bench-2026-10-15-6f091831ab60. Source payload hash: 6f091831ab607a26d65d2ade18cba7ce345ea3a67297b8c378e8a06ce398360d. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-snap:1.1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-swebench-multimodal","area":"coding","label":{"de":"SWE-bench Multimodal","en":"SWE-bench Multimodal"},"weight":0.004761904761904762,"coverage":1,"score":15.384615384615385,"measurements":[{"observationId":"vals-swebench-multimodal:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal","benchmarkId":"vals-swebench-multimodal","benchmarkVersion":"1","comparisonCohort":"vals-swebench-multimodal:1:overall","retrievedAt":"2026-10-15","value":39.167,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":15.384615384615385,"participants":14,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-10-13. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 1032.464. Standardfehler: 2.23. Kosten je Aufgabe in USD: 0.952593. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-swebench-multimodal-2026-10-15-e1651b256650. Prüfsumme der Quelldaten: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-10-13. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 1032.464. Standard error: 2.23. Cost per task in USD: 0.952593. Source ID: vals-ai. Evidence capture ID: vals-swebench-multimodal-2026-10-15-e1651b256650. Source payload hash: e1651b256650426968751f35c0dc448d6f5f626ad02f0ecb13ec62e0b6b6f7a7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-swebench-multimodal:1:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-agent-bench","area":"finance","label":{"de":"Tax Agent Bench","en":"Tax Agent Bench"},"weight":0.01020408163265306,"coverage":1,"score":76.36363636363636,"measurements":[{"observationId":"vals-tax-agent-bench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","benchmarkId":"vals-tax-agent-bench","benchmarkVersion":"1","comparisonCohort":"vals-tax-agent-bench:1:overall","retrievedAt":"2026-10-15","value":32.095,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":76.36363636363636,"participants":56,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 2436.106. Standardfehler: 3.367. Kosten je Aufgabe in USD: 1.552558. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_agent_bench-2026-10-15-05d520e65520. Prüfsumme der Quelldaten: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 2436.106. Standard error: 3.367. Cost per task in USD: 1.552558. Source ID: vals-ai. Evidence capture ID: vals-tax_agent_bench-2026-10-15-05d520e65520. Source payload hash: 05d520e65520cb13dbba02402d163477cc798ab3f0578f42af6ea7e28e213f1f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-agent-bench:1:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-tax-eval-v2","area":"finance","label":{"de":"TaxEval v2","en":"TaxEval v2"},"weight":0.01020408163265306,"coverage":1,"score":88.15789473684211,"measurements":[{"observationId":"vals-tax-eval-v2:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","benchmarkId":"vals-tax-eval-v2","benchmarkVersion":"2","comparisonCohort":"vals-tax-eval-v2:2:overall","retrievedAt":"2026-10-15","value":75.552,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":88.15789473684211,"participants":77,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-09-01. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 265.751. Standardfehler: 0.838. Kosten je Aufgabe in USD: 0.07645. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Prüfsumme der Quelldaten: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-09-01. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 265.751. Standard error: 0.838. Cost per task in USD: 0.07645. Source ID: vals-ai. Evidence capture ID: vals-tax_eval_v2-2026-10-15-8b354a6ae848. Source payload hash: 8b354a6ae848f5bb25a2e8b061b5f576d5b9e0c11b941dc1e6bed078e6529cd6. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-tax-eval-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vcb-1-100","area":"coding","label":{"de":"Vibe Code Bench 1-100","en":"Vibe Code Bench 1-100"},"weight":0.004761904761904762,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"vals-vcb-1-100:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","benchmarkId":"vals-vcb-1-100","benchmarkVersion":"1.0","comparisonCohort":"vals-vcb-1-100:1.0:overall","retrievedAt":"2026-10-15","value":12.833,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":14.285714285714286,"participants":22,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-14. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 5735.815. Standardfehler: 3.183. Kosten je Aufgabe in USD: 9.675734. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Prüfsumme der Quelldaten: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-14. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 5735.815. Standard error: 3.183. Cost per task in USD: 9.675734. Source ID: vals-ai. Evidence capture ID: vals-vcb-1-100-2026-10-15-4a3ebdcd9364. Source payload hash: 4a3ebdcd9364ec3a21193d00de0bf723384d85187244abff859527fd3cc51891. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vcb-1-100:1.0:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"vals-vibe-code-bench","area":"coding","label":{"de":"Vibe Code Bench v1.1","en":"Vibe Code Bench v1.1"},"weight":0.004761904761904762,"coverage":1,"score":56.94444444444444,"measurements":[{"observationId":"vals-vibe-code-bench:overall:alibaba/qwen3.8-max","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","benchmarkId":"vals-vibe-code-bench","benchmarkVersion":"1.1","comparisonCohort":"vals-vibe-code-bench:1.1:overall","retrievedAt":"2026-10-15","value":64.696,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":56.94444444444444,"participants":73,"methodology":{"de":"Kennung des Quellmodells: alibaba/qwen3.8-max. Name des Quellmodells: Qwen 3.8 Max 0902. Anbieter laut Quelle: Alibaba. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-10-12. Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000. Latenz in Sekunden: 10135.139. Standardfehler: 5.362. Kosten je Aufgabe in USD: 8.238852. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-vibe-code-2026-10-15-59a78d2bab11. Prüfsumme der Quelldaten: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: alibaba/qwen3.8-max. Source model name: Qwen 3.8 Max 0902. Provider reported by source: Alibaba. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-10-12. Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000. Latency in seconds: 10135.139. Standard error: 5.362. Cost per task in USD: 8.238852. Source ID: vals-ai. Evidence capture ID: vals-vibe-code-2026-10-15-59a78d2bab11. Source payload hash: 59a78d2bab1196accf0345adffb2b281d01f2520db9ed2e61fd6ab9c2e98221f. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreRunConfiguration":{"de":"Harness: OpenHands. Temperatur: 0.7. Maximale Ausgabetoken: 128000","en":"Harness: OpenHands. Temperature: 0.7. Maximum output tokens: 128000"},"scoreComparisonSetup":{"publicationCohort":"vals-vibe-code-bench:1.1:overall","datasetType":"private","harness":"OpenHands","agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":57.86748876320653,"coveredFamilies":42,"totalFamilies":71,"sourceCount":3,"coverage":0.5255668934240364,"lowerBound":30.413236299528787,"upperBound":77.85654695712515,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":55.222931441153705,"coveredFamilies":12,"totalFamilies":15,"sourceCount":3,"coverage":0.7555555555555555,"lowerBound":41.72399264442724,"upperBound":66.16843708887168,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":4,"totalFamilies":7,"sourceCount":2,"coverage":0.42857142857142855,"lowerBound":34.373280762882814,"upperBound":91.51613790573995,"provisional":true,"configurationsDiffer":true},"math":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.025641025641026,"upperBound":91.02564102564102,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":75.66518847006651,"coveredFamilies":2,"totalFamilies":3,"sourceCount":2,"coverage":0.6666666666666666,"lowerBound":50.44345898004434,"upperBound":83.77679231337768,"provisional":true,"configurationsDiffer":true},"agentic":{"score":70.60467530344916,"coveredFamilies":7,"totalFamilies":12,"sourceCount":2,"coverage":0.5833333333333334,"lowerBound":41.186060593678675,"upperBound":82.85272726034535,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":6,"upperBound":86,"provisional":true,"configurationsDiffer":true},"finance":{"score":62.31643721942563,"coveredFamilies":6,"totalFamilies":7,"sourceCount":1,"coverage":0.8571428571428571,"lowerBound":53.414089045221964,"upperBound":67.69980333093625,"provisional":true,"configurationsDiffer":true},"legal":{"score":74.45004904021299,"coveredFamilies":3,"totalFamilies":4,"sourceCount":1,"coverage":0.75,"lowerBound":55.83753678015974,"upperBound":80.83753678015974,"provisional":true,"configurationsDiffer":true},"health":{"score":50.241439859525904,"coveredFamilies":2,"totalFamilies":3,"sourceCount":1,"coverage":0.6666666666666666,"lowerBound":33.494293239683934,"upperBound":66.82762657301727,"provisional":true,"configurationsDiffer":true},"cybersecurity":{"score":null,"coveredFamilies":1,"totalFamilies":1,"sourceCount":1,"coverage":1,"lowerBound":0,"upperBound":0,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":2,"coverage":0.25,"lowerBound":20.065539513677813,"upperBound":95.0655395136778,"provisional":true,"configurationsDiffer":true},"productivity":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":36.8421052631579,"upperBound":86.84210526315789,"provisional":true,"configurationsDiffer":true},"education":{"score":null,"coveredFamilies":1,"totalFamilies":2,"sourceCount":1,"coverage":0.5,"lowerBound":41.37931034482759,"upperBound":91.37931034482759,"provisional":true,"configurationsDiffer":true}}},{"modelId":"qwen3.8-omni-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"reflection-beam","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ring-2.6-1t","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":34.146341463414636,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":658.17,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":34.146341463414636,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":27.4390243902439,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":70,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":27.4390243902439,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 70,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 70.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.7178659181731186,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,7 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.7%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":3.8,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 3,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 3.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":26.829268292682926,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":746.17,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":26.829268292682926,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":39.130434782608695,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":44.5578231292517,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":39.130434782608695,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 44,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 44.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":23.170731707317074,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-37.7,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":23.170731707317074,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":28.048780487804876,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":21.5940685820204,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":28.048780487804876,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 21,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 21.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":40.853658536585364,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Name des Quellmodells: Ring 2.6 1T. Anbieter laut Quelle: InclusionAI. Modellstand: Ring-2.6-1T. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c/ring-2-6-1t/ring-2-6-1t. Source model name: Ring 2.6 1T. Provider reported by source: InclusionAI. Model snapshot: Ring-2.6-1T. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Ring-2.6-1T (Release ring-2-6-1t, Zeile ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Ring-2.6-1T (release ring-2-6-1t, row ring-2-6-1t, UUID eb4ba465-3fcd-4065-9fe2-e8225e7b2c6c, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":1,"coverage":0.083843537414966,"lowerBound":2.3175510925473057,"upperBound":93.93319735105071,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":1.3617886178861793,"upperBound":98.02845528455285,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":9.233449477351915,"upperBound":73.5191637630662,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":7.723577235772359,"upperBound":74.39024390243902,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":6.300813008130081,"upperBound":81.30081300813008,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":7.826086956521738,"upperBound":87.82608695652173,"provisional":true,"configurationsDiffer":true}}},{"modelId":"sarvam-sarvam-105b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"sarvam-sarvam-30b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"sdaia-allam-2-7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-1-6","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-1-6-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-1-8","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-2.0-code","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-2.0-lite","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-2.0-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-2.0-pro","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"seed-2.1-turbo","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"solar-mini","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"solar-mini-4","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"solar-pro-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"solar-pro-3","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":98.98,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":8.536585365853659,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":32.3333333333333,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":8.536585365853659,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 32,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 32.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.49734242022629577,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":14.024390243902438,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.6,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.024390243902438,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":264.94,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":69.56521739130434,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":71.2018140589569,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":69.56521739130434,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 71,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 71.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":7.317073170731708,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-53.4,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":7.317073170731708,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":10.2873030583874,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 10,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 10.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.5,"score":17.682926829268293,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b89c4faf-219e-4171-a1aa-e3bd2fd0a924","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":17.682926829268293,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Name des Quellmodells: Solar Pro 3. Anbieter laut Quelle: Upstage. Modellstand: Solar Pro 3. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b89c4faf-219e-4171-a1aa-e3bd2fd0a924/solar-pro-3/solar-pro-3. Source model name: Solar Pro 3. Provider reported by source: Upstage. Model snapshot: Solar Pro 3. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Solar Pro 3 (Release solar-pro-3, Zeile solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Solar Pro 3 (release solar-pro-3, row solar-pro-3, UUID b89c4faf-219e-4171-a1aa-e3bd2fd0a924, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":9,"totalFamilies":71,"sourceCount":1,"coverage":0.083843537414966,"lowerBound":1.689201131141746,"upperBound":93.30484738964515,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":1,"coverage":0.03333333333333335,"lowerBound":0.5894308943089434,"upperBound":97.25609756097562,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":4.268292682926829,"upperBound":68.55400696864112,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":2.4390243902439024,"upperBound":69.10569105691057,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":2.439024390243903,"upperBound":77.4390243902439,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":13.91304347826087,"upperBound":93.91304347826087,"provisional":true,"configurationsDiffer":true}}},{"modelId":"solar-pro-4","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"spark-x2.5","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"spark-x2.5-1.7b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"spark-x2.5-4b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"step-3.5-flash","contributions":[{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":50.72463768115942,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:c8158c23-6fff-4c31-911d-954c32d80c28","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":64.5578231292517,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":50.72463768115942,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Step 3.5 Flash (Release step-3-5-flash-0202, Zeile step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/c8158c23-6fff-4c31-911d-954c32d80c28/step-3-5-flash-0202/step-3-5-flash-0202. Name des Quellmodells: Step 3.5 Flash. Anbieter laut Quelle: StepFun. Modellstand: Step 3.5 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 64,6 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Step 3.5 Flash (release step-3-5-flash-0202, row step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/c8158c23-6fff-4c31-911d-954c32d80c28/step-3-5-flash-0202/step-3-5-flash-0202. Source model name: Step 3.5 Flash. Provider reported by source: StepFun. Model snapshot: Step 3.5 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 64.6%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Step 3.5 Flash (Release step-3-5-flash-0202, Zeile step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.5 Flash (release step-3-5-flash-0202, row step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Step 3.5 Flash (Release step-3-5-flash-0202, Zeile step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.5 Flash (release step-3-5-flash-0202, row step-3-5-flash-0202, UUID c8158c23-6fff-4c31-911d-954c32d80c28, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":1,"totalFamilies":71,"sourceCount":1,"coverage":0.014285714285714287,"lowerBound":0.7246376811594204,"upperBound":99.296066252588,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":10.144927536231885,"upperBound":90.14492753623189,"provisional":true,"configurationsDiffer":true}}},{"modelId":"step-3.7-flash","contributions":[{"familyId":"artificial-analysis-apex-agents-aa","area":"agentic","label":{"de":"APEX-Agents-AA","en":"APEX-Agents-AA"},"weight":0.005952380952380952,"coverage":1,"score":30.76923076923077,"measurements":[{"observationId":"aa:artificial-analysis-apex-agents-aa:6c7b322e-2f35-48ff-9171-fb621a726fc0","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","benchmarkId":"artificial-analysis-apex-agents-aa","benchmarkVersion":"AA public cohort: 452 tasks","comparisonCohort":"artificial-analysis-apex-agents-aa:AA public cohort: 452 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":14.823008849557501,"unit":"%","metric":"Pass@1 success rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":30.76923076923077,"participants":27,"methodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Kennung des Quellmodells: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Name des Quellmodells: Step 3.7 Flash. Anbieter laut Quelle: StepFun. Modellstand: Step 3.7 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 14,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Source model ID: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Source model name: Step 3.7 Flash. Provider reported by source: StepFun. Model snapshot: Step 3.7 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled. Score display: 14.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n452 Aufgaben aus der öffentlichen 480-Aufgaben-Version, drei Wiederholungen pro Aufgabe. Ein Durchlauf besteht nur, wenn alle Kriterien der Rubrik erfüllt sind; der Leaderboardwert mittelt pass@1 über die Wiederholungen.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: 200-turn cap; MCP-backed workplace tools in Archipelago. Versuche: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n452 tasks from the public 480-task release, with three repeats per task. A run passes only if every rubric criterion is satisfied; the leaderboard averages pass@1 across repeats.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: 200-turn cap; MCP-backed workplace tools in Archipelago. Attempts: pass@1, averaged over 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-apex-agents-aa\",\"AA public cohort: 452 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"200-turn cap; MCP-backed workplace tools in Archipelago","attempts":"pass@1, averaged over 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":57.971014492753625,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:6c7b322e-2f35-48ff-9171-fb621a726fc0","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":67.2789115646258,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":57.971014492753625,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Name des Quellmodells: Step 3.7 Flash. Anbieter laut Quelle: StepFun. Modellstand: Step 3.7 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 67,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Source model name: Step 3.7 Flash. Provider reported by source: StepFun. Model snapshot: Step 3.7 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 67.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-itbench-aa","area":"agentic","label":{"de":"ITBench-AA (SRE)","en":"ITBench-AA (SRE)"},"weight":0.005952380952380952,"coverage":1,"score":14.285714285714286,"measurements":[{"observationId":"aa:artificial-analysis-itbench-aa:6c7b322e-2f35-48ff-9171-fb621a726fc0","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","benchmarkId":"artificial-analysis-itbench-aa","benchmarkVersion":"AA cohort: 59 tasks","comparisonCohort":"artificial-analysis-itbench-aa:AA cohort: 59 tasks:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":30.2730696798493,"unit":"%","metric":"Average precision at full recall","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.285714285714286,"participants":36,"methodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Kennung des Quellmodells: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Name des Quellmodells: Step 3.7 Flash. Anbieter laut Quelle: StepFun. Modellstand: Step 3.7 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Angezeigter Wert: 30,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Source model ID: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Source model name: Step 3.7 Flash. Provider reported by source: StepFun. Model snapshot: Step 3.7 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled. Score display: 30.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\n59 Kubernetes-Incidentaufgaben, davon 40 öffentliche und 19 private, mit drei Wiederholungen je Aufgabe. Ein Lauf erhält null, wenn eine tatsächliche Ursache fehlt, sonst die Präzision bei vollständigem Recall. Dies ist nur die SRE-Teilmenge von ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Werkzeugrichtlinie: Shell access to an offline Kubernetes incident snapshot. Versuche: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\n59 Kubernetes incident tasks, 40 public and 19 private, with three repeats per task. A run scores zero if it misses any true root cause; otherwise it receives its precision at full recall. This is the SRE slice of ITBench.. Harness: Artificial Analysis Stirrup. Agent: Stirrup agent. Tool policy: Shell access to an offline Kubernetes incident snapshot. Attempts: Average precision at full recall, 3 repeats per task. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-itbench-aa\",\"AA cohort: 59 tasks\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":"Artificial Analysis Stirrup","agent":"Stirrup agent","toolPolicy":"Shell access to an offline Kubernetes incident snapshot","attempts":"Average precision at full recall, 3 repeats per task"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"mmmu-pro","area":"multimodal","label":{"de":"MMMU-Pro","en":"MMMU-Pro"},"weight":0.017857142857142856,"coverage":0.5,"score":44.642857142857146,"measurements":[{"observationId":"aa:artificial-analysis-mmmu-pro:6c7b322e-2f35-48ff-9171-fb621a726fc0","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","benchmarkId":"artificial-analysis-mmmu-pro","benchmarkVersion":"AA cohort: 1,730 questions","comparisonCohort":"artificial-analysis-mmmu-pro:AA cohort: 1,730 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":75.3179190751445,"unit":"%","metric":"pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":44.642857142857146,"participants":57,"methodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Kennung des Quellmodells: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Name des Quellmodells: Step 3.7 Flash. Anbieter laut Quelle: StepFun. Modellstand: Step 3.7 Flash. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled. Angezeigter Wert: 75,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Source model ID: artificial-analysis/6c7b322e-2f35-48ff-9171-fb621a726fc0/step-3-7-flash/step-3-7-flash. Source model name: Step 3.7 Flash. Provider reported by source: StepFun. Model snapshot: Step 3.7 Flash. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled. Score display: 75.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Step 3.7 Flash (Release step-3-7-flash, Zeile step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nDer AA-Lauf umfasst 1.730 Fragen mit zehn Antwortoptionen. Die Antwort wird per regulärem Ausdruck extrahiert und als pass@1 bewertet.. Werkzeugrichtlinie: Ten-option multiple choice; regex answer extraction. Versuche: pass@1, one run per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Step 3.7 Flash (release step-3-7-flash, row step-3-7-flash, UUID 6c7b322e-2f35-48ff-9171-fb621a726fc0, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nThe AA run covers 1,730 questions with ten answer options. Answers are extracted by regex and scored pass@1.. Tool policy: Ten-option multiple choice; regex answer extraction. Attempts: pass@1, one run per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-mmmu-pro\",\"AA cohort: 1,730 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Ten-option multiple choice; regex answer extraction","attempts":"pass@1, one run per question"},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":4,"totalFamilies":71,"sourceCount":1,"coverage":0.035119047619047626,"lowerBound":1.494938485427616,"upperBound":97.98303372352285,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-19","retrievedThrough":"2026-10-19","areas":{"agentic":{"score":null,"coveredFamilies":2,"totalFamilies":12,"sourceCount":1,"coverage":0.16666666666666666,"lowerBound":3.7545787545787546,"upperBound":87.0879120879121,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":11.594202898550726,"upperBound":91.59420289855072,"provisional":true,"configurationsDiffer":true},"multimodal":{"score":null,"coveredFamilies":1,"totalFamilies":4,"sourceCount":1,"coverage":0.125,"lowerBound":5.580357142857143,"upperBound":93.08035714285714,"provisional":true,"configurationsDiffer":true}}},{"modelId":"step-5-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"stepfun-step-3.5-flash-2603","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"swe-2","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"swiss-ai-apertus-70b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"swiss-ai-apertus-8b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tencent-hy3-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tencent-hy4-preview","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"ternary-bonsai-2-27b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tiny-aya-earth","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tiny-aya-fire","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tiny-aya-global","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"tiny-aya-water","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"trendyol-asure-12b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"trinity-large-thinking","contributions":[{"familyId":"artificial-analysis-aa-briefcase","area":"agentic","label":{"de":"AA-Briefcase","en":"AA-Briefcase"},"weight":0.005952380952380952,"coverage":1,"score":18.29268292682927,"measurements":[{"observationId":"aa:artificial-analysis-aa-briefcase:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase","benchmarkId":"artificial-analysis-aa-briefcase","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-briefcase:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":255.45,"unit":"Elo","metric":"AA-Briefcase Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":18.29268292682927,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-briefcase\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-aa-lcr","area":"reasoning","label":{"de":"AA-LCR","en":"AA-LCR"},"weight":0.01020408163265306,"coverage":1,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-aa-lcr:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","benchmarkId":"artificial-analysis-aa-lcr","benchmarkVersion":"v1.1","comparisonCohort":"artificial-analysis-aa-lcr:v1.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":38,"unit":"%","metric":"Pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 38,0 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 38.0%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-aa-lcr\",\"v1.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-automationbench-aa","area":"agentic","label":{"de":"AutomationBench-AA","en":"AutomationBench-AA"},"weight":0.005952380952380952,"coverage":1,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-automationbench-aa:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","benchmarkId":"artificial-analysis-automationbench-aa","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-automationbench-aa:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.3069234137838786,"unit":"%","metric":"Objectives completed without guardrail violations","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-automationbench-aa\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdp-pdf","area":"reasoning","label":{"de":"GDP.pdf","en":"GDP.pdf"},"weight":0.01020408163265306,"coverage":1,"score":10.975609756097562,"measurements":[{"observationId":"aa:artificial-analysis-gdp-pdf:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","benchmarkId":"artificial-analysis-gdp-pdf","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-gdp-pdf:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":1.2,"unit":"%","metric":"All-pass rate","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":10.975609756097562,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 1,2 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 1.2%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdp-pdf\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-gdpval-aa","area":"agentic","label":{"de":"GDPval-AA","en":"GDPval-AA"},"weight":0.005952380952380952,"coverage":1,"score":12.195121951219512,"measurements":[{"observationId":"aa:artificial-analysis-gdpval-aa:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","benchmarkId":"artificial-analysis-gdpval-aa","benchmarkVersion":"v2.1","comparisonCohort":"artificial-analysis-gdpval-aa:v2.1:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":336.63,"unit":"Elo","metric":"GDPval-AA Elo","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":12.195121951219512,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-gdpval-aa\",\"v2.1\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-ifbench","area":"general","label":{"de":"IFBench","en":"IFBench"},"weight":0.014285714285714285,"coverage":1,"score":43.47826086956522,"measurements":[{"observationId":"aa:artificial-analysis-ifbench:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","benchmarkId":"artificial-analysis-ifbench","benchmarkVersion":"AA test set: 294 questions","comparisonCohort":"artificial-analysis-ifbench:AA test set: 294 questions:standalone:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":56.25850340136051,"unit":"%","metric":"IFBench prompt-level loose accuracy (pass@1)","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":43.47826086956522,"participants":70,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Angezeigter Wert: 56,3 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled. Score display: 56.3%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.\nArtificial Analysis führt 294 Fragen aus dem IFBench-Testdatensatz mit fünf Wiederholungen je Frage aus. Der offizielle Evaluator bewertet die Genauigkeit je Prompt im Modus loose als pass@1.. Werkzeugrichtlinie: Single-turn answer evaluated with the official IFBench source code. Versuche: pass@1, 5 repeats per question. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.\nArtificial Analysis runs 294 questions from the IFBench test set with five repeats per question. The official evaluator scores prompt-level accuracy in loose mode as pass@1.. Tool policy: Single-turn answer evaluated with the official IFBench source code. Attempts: pass@1, 5 repeats per question. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-ifbench\",\"AA test set: 294 questions\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":"Single-turn answer evaluated with the official IFBench source code","attempts":"pass@1, 5 repeats per question"},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"artificial-analysis-omniscience","area":"knowledge","label":{"de":"AA-Omniscience","en":"AA-Omniscience"},"weight":0.023809523809523808,"coverage":1,"score":14.634146341463415,"measurements":[{"observationId":"aa:artificial-analysis-omniscience:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","benchmarkId":"artificial-analysis-omniscience","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-omniscience:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":-44.06666666666667,"unit":"points","metric":"AA-Omniscience Index","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":14.634146341463415,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-omniscience\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]},{"familyId":"humanitys-last-exam","area":"reasoning","label":{"de":"Humanity’s Last Exam","en":"Humanity’s Last Exam"},"weight":0.01020408163265306,"coverage":0.5,"score":21.951219512195124,"measurements":[{"observationId":"aa:artificial-analysis-humanitys-last-exam:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/humanitys-last-exam","benchmarkId":"artificial-analysis-humanitys-last-exam","benchmarkVersion":"unspecified","comparisonCohort":"artificial-analysis-humanitys-last-exam:unspecified:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":15.8480074142725,"unit":"%","metric":"Text-only pass@1 accuracy","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":21.951219512195124,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 15,8 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 15.8%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-humanitys-last-exam\",\"unspecified\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true}]},{"familyId":"terminal-bench","area":"coding","label":{"de":"Terminal-Bench","en":"Terminal-Bench"},"weight":0.004761904761904762,"coverage":0.6666666666666666,"score":33.02119628339141,"measurements":[{"observationId":"aa:artificial-analysis-terminal-bench-4:b6d2e43d-3082-43f5-9318-0f4dbcb54163","sourceId":"artificial-analysis","sourceName":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","benchmarkId":"artificial-analysis-terminal-bench-4","benchmarkVersion":"4.0","comparisonCohort":"artificial-analysis-terminal-bench-4:4.0:v4.3.2:reasoning:tokens=2000:suffix=unspecified","retrievedAt":"2026-10-19","value":0.505050505050505,"unit":"%","metric":"Task success","task":"overall","evaluator":"Artificial Analysis","higherIsBetter":true,"percentile":40.853658536585364,"participants":83,"methodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Kennung des Quellmodells: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee AI. Modellstand: Trinity Large Thinking. Evaluator: Artificial Analysis. Datensatztyp: artificial-analysis-evaluation. Reasoning: reasoning enabled. Angezeigter Wert: 0,5 %. Quellenkennung: artificial-analysis. Kennung der Evidenzaufnahme: aa-model-page-2026-10-19-8a022369738c. Prüfsumme der Quelldaten: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parserversion: artificial-analysis-rsc-models-v2. Importiert am: 2026-10-19","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Source model ID: artificial-analysis/b6d2e43d-3082-43f5-9318-0f4dbcb54163/trinity-large-thinking/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee AI. Model snapshot: Trinity Large Thinking. Evaluator: Artificial Analysis. Dataset type: artificial-analysis-evaluation. Reasoning: reasoning enabled. Score display: 0.5%. Source ID: artificial-analysis. Evidence capture ID: aa-model-page-2026-10-19-8a022369738c. Source payload hash: 8a022369738cbe2a0e2f9bf486a2882865dbb0c6f1396dfec84059a802560e3c. Parser version: artificial-analysis-rsc-models-v2. Imported at: 2026-10-19"},"comparisonMethodology":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreRunConfiguration":{"de":"Exakte AA-Konfiguration: Trinity Large Thinking (Release trinity-large-thinking, Zeile trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=nicht angegeben). Der konfigurierte Wert ist kein gemessener Tokenverbrauch.. Reasoning: reasoning enabled","en":"Exact AA configuration: Trinity Large Thinking (release trinity-large-thinking, row trinity-large-thinking, UUID b6d2e43d-3082-43f5-9318-0f4dbcb54163, reasoning=true, source reasoning-token configuration budget=2000, suffix=not specified). This configured value is not measured token consumption.. Reasoning: reasoning enabled"},"scoreComparisonSetup":{"publicationCohort":"[\"artificial-analysis-terminal-bench-4\",\"4.0\",\"aa-model-page-2026-10-19-8a022369738c\"]","datasetType":"artificial-analysis-evaluation","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.5,"configurationsDiffer":true},{"observationId":"vals-terminal-bench-2:overall:arcee-ai/trinity-large-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","benchmarkId":"vals-terminal-bench-2","benchmarkVersion":"2.0","comparisonCohort":"vals-terminal-bench-2:2.0:overall","retrievedAt":"2026-10-15","value":17.978,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":9.523809523809524,"participants":43,"methodology":{"de":"Kennung des Quellmodells: arcee-ai/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee-Ai. Evaluator: Vals AI. Datensatztyp: public. Stand der Rangliste: 2026-06-04. Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 1015.224. Standardfehler: 4.093. Kosten je Aufgabe in USD: 0.325546. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-terminal-bench-2-2026-10-15-38e640d99525. Prüfsumme der Quelldaten: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: arcee-ai/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee-Ai. Evaluator: Vals AI. Dataset type: public. Leaderboard date: 2026-06-04. Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 1015.224. Standard error: 4.093. Cost per task in USD: 0.325546. Source ID: vals-ai. Evidence capture ID: vals-terminal-bench-2-2026-10-15-38e640d99525. Source payload hash: 38e640d995259778e531f048b2f2c7b3b281cb05b56be643ddc9dc53eb85adf7. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-terminal-bench-2:2.0:overall","datasetType":"public","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":0.16666666666666666,"configurationsDiffer":true}]},{"familyId":"vals-corp-fin-v2","area":"finance","label":{"de":"CorpFin v2","en":"CorpFin v2"},"weight":0.01020408163265306,"coverage":1,"score":18.91891891891892,"measurements":[{"observationId":"vals-corp-fin-v2:overall:arcee-ai/trinity-large-thinking","sourceId":"vals-ai","sourceName":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","benchmarkId":"vals-corp-fin-v2","benchmarkVersion":"2","comparisonCohort":"vals-corp-fin-v2:2:overall","retrievedAt":"2026-10-15","value":54.662,"unit":"%","metric":"accuracy","task":"overall","evaluator":"Vals AI","higherIsBetter":true,"percentile":18.91891891891892,"participants":75,"methodology":{"de":"Kennung des Quellmodells: arcee-ai/trinity-large-thinking. Name des Quellmodells: Trinity Large Thinking. Anbieter laut Quelle: Arcee-Ai. Evaluator: Vals AI. Datensatztyp: private. Stand der Rangliste: 2026-08-12. Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000. Latenz in Sekunden: 12.482. Standardfehler: 0.977. Kosten je Aufgabe in USD: 0.01939. Quellenkennung: vals-ai. Kennung der Evidenzaufnahme: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Prüfsumme der Quelldaten: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parserversion: vals-astro-v3. Importiert am: 2026-10-15","en":"Source model ID: arcee-ai/trinity-large-thinking. Source model name: Trinity Large Thinking. Provider reported by source: Arcee-Ai. Evaluator: Vals AI. Dataset type: private. Leaderboard date: 2026-08-12. Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000. Latency in seconds: 12.482. Standard error: 0.977. Cost per task in USD: 0.01939. Source ID: vals-ai. Evidence capture ID: vals-corp_fin_v2-2026-10-15-7cf2217ab372. Source payload hash: 7cf2217ab3728d5a84bffce82aab9fc52d7f810b0cbe8c65b387df47f3d31c9b. Parser version: vals-astro-v3. Imported at: 2026-10-15"},"comparisonMethodology":{"de":"Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000"},"scoreRunConfiguration":{"de":"Temperatur: 0.3. Top P: 0.95. Maximale Ausgabetoken: 80000","en":"Temperature: 0.3. Top P: 0.95. Maximum output tokens: 80000"},"scoreComparisonSetup":{"publicationCohort":"vals-corp-fin-v2:2:overall","datasetType":"private","harness":null,"agent":null,"toolPolicy":null,"attempts":null},"scoreReasoningEffort":null,"weight":1,"configurationsDiffer":true}]}],"overall":{"score":null,"coveredFamilies":10,"totalFamilies":71,"sourceCount":2,"coverage":0.09484126984126985,"lowerBound":1.9279985614631883,"upperBound":92.4438715773362,"provisional":true,"configurationsDiffer":true},"retrievedFrom":"2026-10-15","retrievedThrough":"2026-10-19","areas":{"coding":{"score":null,"coveredFamilies":1,"totalFamilies":15,"sourceCount":2,"coverage":0.04444444444444445,"lowerBound":1.4676087237062856,"upperBound":97.02316427926183,"provisional":true,"configurationsDiffer":true},"reasoning":{"score":null,"coveredFamilies":3,"totalFamilies":7,"sourceCount":1,"coverage":0.3571428571428571,"lowerBound":4.878048780487806,"upperBound":69.1637630662021,"provisional":true,"configurationsDiffer":true},"knowledge":{"score":null,"coveredFamilies":1,"totalFamilies":3,"sourceCount":1,"coverage":0.3333333333333333,"lowerBound":4.878048780487805,"upperBound":71.54471544715447,"provisional":true,"configurationsDiffer":true},"agentic":{"score":null,"coveredFamilies":3,"totalFamilies":12,"sourceCount":1,"coverage":0.25,"lowerBound":4.369918699186992,"upperBound":79.369918699187,"provisional":true,"configurationsDiffer":true},"general":{"score":null,"coveredFamilies":1,"totalFamilies":5,"sourceCount":1,"coverage":0.2,"lowerBound":8.695652173913043,"upperBound":88.69565217391305,"provisional":true,"configurationsDiffer":true},"finance":{"score":null,"coveredFamilies":1,"totalFamilies":7,"sourceCount":1,"coverage":0.14285714285714285,"lowerBound":2.7027027027027026,"upperBound":88.41698841698843,"provisional":true,"configurationsDiffer":true}}},{"modelId":"virtuoso-large","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"voxtral-small-24b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"xiaomi-mimo-v2-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"xing4-0-29b-a4b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"zdtaichu5-0-9b","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"zhipuai-glm-4.6v-flash","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}},{"modelId":"zhipuai-glm-4.7-flashx","contributions":[],"overall":{"score":null,"coveredFamilies":0,"totalFamilies":71,"sourceCount":0,"coverage":0,"lowerBound":0,"upperBound":100,"provisional":true,"configurationsDiffer":false},"retrievedFrom":null,"retrievedThrough":null,"areas":{}}]}